Publications

@conference{Liang2025,

title = {DIFFER: Disentangling Identity Features via Semantic Cues for Clothes-Changing Person Re-ID},

author = {Xin Liang and Yogesh Rawat},

url = {https://cvpr.thecvf.com/virtual/2025/poster/33555

https://openreview.net/pdf?id=tI3ZlbEhOM

https://openreview.net/attachment?id=tI3ZlbEhOM&name=supplementary_material},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this work, we focus on clothes-changing person re-identification (CC-ReID), which aims to recognize individuals under different clothing scenarios. Current CC-ReID approaches either concentrate on modeling body shape using additional modalities including silhouette, pose, and body mesh, potentially causing the model to overlook other critical biometric traits such as gender, age, and style, or they incorporate supervision through additional labels that the model tries to disregard or emphasize, such as clothing or personal attributes. However, these annotations are discrete in nature and do not capture comprehensive descriptions.In this work, we propose DIFFER: Disentangle Identity Features From Entangled Representations, a novel adversarial learning method that leverages textual descriptions to disentangle identity features. Recognizing that image features inherently mix inseparable information, DIFFER introduces NBDetach, a mechanism that utilizes the separable nature of text descriptions as disentanglement supervision to partition the feature space into distinct subspaces, enabling the effective separation of identity-related features from non-biometric features through gradient reversal. We evaluate DIFFER on 4 different benchmark datasets (LTCC, PRCC, CelebreID-Light, and CCVID) to demonstrate its effectiveness and provide state-of-the-art performance across all the benchmarks. DIFFER consistently outperforms the baseline method, with improvements in top-1 accuracy of 3.6% on LTCC, 3.4% on PRCC, 2.5% on CelebReID-Light, and 1% on CCVID. The code will be made publicly available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2025,

title = {HierarQ: Task-Aware Hierarchical Q-Former for Enhanced Video Understanding},

author = {Shehreen Azad and Vibhav Vineet and Yogesh Rawat},

url = {https://arxiv.org/abs/2503.08585

},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite advancements in multimodal large language models (MLLMs), current approaches struggle in medium-to-long video understanding due to frame and context length limitations. As a result, these models often depend on frame sampling, which risks missing key information over time and lacks task-specific relevance. To address these challenges, we introduce HierarQ, a task-aware hierarchical Q-Former based framework that sequentially processes frames to bypass the need for frame sampling, while avoiding LLM's context length limitations. We introduce a lightweight two-stream language-guided feature modulator to incorporate task awareness in video understanding, with the entity stream capturing frame-level object information within a short context and the scene stream identifying their broader interactions over longer period of time. Each stream is supported by dedicated memory banks which enables our proposed Hierachical Querying transformer (HierarQ) to effectively capture short and long-term context. Extensive evaluations on 10 video benchmarks across video understanding, question answering, and captioning tasks demonstrate HierarQ's state-of-the-art performance across most datasets, proving its robustness and efficiency for comprehensive video analysis.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Garg2025,

title = {STPro: Spatial and Temporal Progressive Learning for Weakly Supervised Spatio-Temporal Grounding},

author = {Aaryan Garg and Akash Kumar and Yogesh Rawat},

url = {https://aaryangrg.github.io/research/stpro},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vayani2025,

title = {All Languages Matter: Evaluating LMMs on Culturally Diverse 100 Languages},

author = {Ashmal Vayani and Dinura Dissanayake and Hasindri Watawana and Noor Ahsan and Nevasini Sasikumar and Omkar Thawakar and Henok Biadglign Ademtew and Yahya Hmaiti and Amandeep Kumar and Kartik Kuckreja and Mykola Maslych and Wafa Al Ghallabi and Mihail Mihaylov and Chao Qin and Abdelrahman M Shaker and Mike Zhang and Mahardika Krisna Ihsani and Amiel Esplana and Monil Gokani and Shachar Mirkin and Harsh Singh and Ashay Srivastava and Endre Hamerlik and Fathinah Asma Izzati and Fadillah Adamsyah Maani and Sebastian Cavada and Jenny Chim and Rohit Gupta and Sanjay Manjunath and Kamila Zhumakhanova and Feno Heriniaina Rabevohitra and Azril Amirudin and Muhammad Ridzuan and Daniya Kareem and Ketan More and Kunyang Li and Pramesh Shakya and Muhammad Saad and Amirpouya Ghasemaghaei and Amirbek Djanibekov and Dilshod Azizov and Branislava Jankovic and Naman Bhatia and Alvaro Cabrera and Johan Obando-Ceron and Olympiah Otieno and Fabian Farestam and Muztoba Rabbani and Sanoojan Baliah and Santosh Sanjeev and Abduragim Shtanchaev and Maheen Fatima and Thao Nguyen and Amrin Kareem and Toluwani Aremu and Nathan Xavier and Amit Bhatkal and Hawau Toyin and Aman Chadha and Hisham Cholakkal and Rao Muhammad Anwer and Michael Felsberg and Jorma Laaksonen and Thamar Solorio and Monojit Choudhury and Ivan Laptev and Mubarak Shah and Salman Khan and Fahad Khan},

url = {https://arxiv.org/abs/2411.16508

https://mbzuai-oryx.github.io/ALM-Bench/

https://huggingface.co/datasets/MBZUAI/ALM-Bench

https://github.com/mbzuai-oryx/ALM-Bench},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Existing Large Multimodal Models (LMMs) generally focus on only a few regions and languages. As LMMs continue to improve, it is increasingly important to ensure they understand cultural contexts, respect local sensitivities, and support low-resource languages, all while effectively integrating corresponding visual cues. In pursuit of culturally diverse global multimodal models, our proposed All Languages Matter Benchmark (ALM-bench) represents the largest and most comprehensive effort to date for evaluating LMMs across 100 languages. ALM-bench challenges existing models by testing their ability to understand and reason about culturally diverse images paired with text in various languages, including many low-resource languages traditionally underrepresented in LMM research. The benchmark offers a robust and nuanced evaluation framework featuring various question formats, including true/false, multiple choice, and open-ended questions, which are further divided into short and long-answer categories. ALM-bench design ensures a comprehensive assessment of a model's ability to handle varied levels of difficulty in visual and linguistic reasoning. To capture the rich tapestry of global cultures, ALM-bench carefully curates content from 13 distinct cultural aspects, ranging from traditions and rituals to famous personalities and celebrations. Through this, ALM-bench not only provides a rigorous testing ground for state-of-the-art open and closed-source LMMs but also highlights the importance of cultural and linguistic inclusivity, encouraging the development of models that can serve diverse global populations effectively. Our benchmark is publicly available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huynh2025b,

title = {CoLLM:ALargeLanguage Model for Composed Image Retrieva},

author = {Chuong Huynh and Jinyu Yang and Ashish Tawari and Mubarak Shah and Son Tran and Raffay Hamid and Trishul Chilimbi and Abhinav Shrivastava},

url = {https://www.amazon.science/publications/collm-a-large-language-model-for-composed-image-retrieval

https://assets.amazon.science/55/e0/7ead8a494dc49fce25cb9947654c/collm-a-large-language-model-for-composed-image-retrieval.pdf

https://collm-cvpr25.github.io/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a complex task that aims to retrieve images based on a multimodal query. Typical training data consists of triplets containing a reference image, a textual description of desired modifications, and the target image, which are expensive and time-consuming to acquire. The scarcity of CIR datasets has led to zero-shot approaches utilizing synthetic triplets or leveraging vision-language models (VLMs) with ubiquitous web-crawled image-caption pairs. However, these methods have significant limitations: synthetic triplets suffer from limited scale, lack of diversity, and unnatural modification text, while image-caption pairs hinder joint embedding learning of the multimodal query due to the absence of triplet data. Moreover, existing approaches struggle with complex and nuanced modification texts that demand sophisticated fusion and understanding of vision and language modalities. We present CoLLM, a one-stop framework that effectively addresses these limitations. Our approach generates triplets on-the-fly from image-caption pairs, enabling supervised training without manual annotation. We leverage Large Language Models (LLMs) to generate joint embeddings of reference images and modification texts, facilitating deeper multimodal fusion. Additionally, we introduce Multi-Text CIR (MTCIR), a large-scale dataset comprising 3.4M samples, and refine existing CIR benchmarks (CIRR and Fashion-IQ) to enhance evaluation reliability. Experimental results demonstrate that CoLLM achieves state-of0the-art performance across multiple CIR benchmarks and settings. MTCIR yields competitive results, with up to 15% performance improvement. Our refined benchmarks provide more reliable evaluation metrics for CIR models, contributing to the advancement of this important field. Project page is at collm-cvpr25.github.io.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025,

title = {M-LLMBasedVideo Frame Selection for Efficient Video Understanding},

author = {Kai Hu and Feng Gao and Xiaohan Nie and Peng Zhou and Son Tran and Tal Neiman and Lingyun Wang and Mubarak Shah and Raffay Hamid and Bing Yin and Trishul Chilimbi},

url = {https://www.amazon.science/publications/m-llm-based-video-frame-selection-for-efficient-video-understanding

https://assets.amazon.science/1a/4e/6794e1dd4fb2acb30eda428a5119/m-llm-based-video-frame-selection-for-efficient-video-understanding.pdf

},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in Multi-Modal Large Language Models (M-LLMs) show promising results in video reasoning. Popular Multi-Modal Large Language Model (M-LLM) frameworks usually apply naive uniform sampling to reduce the number of video frames that are fed into an M-LLM, particularly for long context videos. However, it could lose crucial context in certain periods of a video, so that the downstream M-LLM may not have sufficient visual information to answer a question. To attack this pain point, we propose a light-weight M-LLM-based frame selection method that adaptively select frames that are more relevant to users’ queries. In order to train the proposed frame selector, we introduce two supervision signals (i) Spatial signal, where single frame importance score by prompting a M-LLM; (ii) Temporal signal, in which multiple frames selection by prompting Large Language Model (LLM) using the captions of all frame candidates. The selected frames are then digested by a frozen downstream video M-LLM for visual reasoning and question answering. Empirical results show that the proposed M-LLM video frame selector improves the performances various downstream video Large Language Model (video-LLM) across medium (ActivityNet, NExT-QA) and long (EgoSchema, LongVideoBench) context video question answering benchmarks.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Croitoru2025,

title = {Curriculum Direct Preference Optimization for Diffusion and Consistency Models},

author = {Florinel-Alin Croitoru and Vlad Hondru and Radu Tudor Ionescu and Nicu Sebe and Mubarak Shah},

url = {https://arxiv.org/abs/2405.13637

https://github.com/CroitoruAlin/Curriculum-DPO





},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Direct Preference Optimization (DPO) has been proposed as an effective and efficient alternative to reinforcement learning from human feedback (RLHF). In this paper, we propose a novel and enhanced version of DPO based on curriculum learning for text-to-image generation. Our method is divided into two training stages. First, a ranking of the examples generated for each prompt is obtained by employing a reward model. Then, increasingly difficult pairs of examples are sampled and provided to a text-to-image generative (diffusion or consistency) model. Generated samples that are far apart in the ranking are considered to form easy pairs, while those that are close in the ranking form hard pairs. In other words, we use the rank difference between samples as a measure of difficulty. The sampled pairs are split into batches according to their difficulty levels, which are gradually used to train the generative model. Our approach, Curriculum DPO, is compared against state-of-the-art fine-tuning approaches on nine benchmarks, outperforming the competing methods in terms of text alignment, aesthetics and human preference. Our code is available at this https URL.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2025,

title = {Generative Zero-Shot Composed Image Retrieval},

author = {Lan Wang and Wei Ao and Vishnu Boddeti and Sernam Lim},

url = {https://hal.cse.msu.edu/papers/cig-generative-zero-shot-composed-image-retrieval/

https://hal.cse.msu.edu/assets/pdfs/papers/2025-cvpr-cig-generative-zero-shot-composed-image-retrieval.pdf

https://hal.cse.msu.edu/assets/pdfs/papers/2025-cvpr-cig-generative-zero-shot-composed-image-retrieval-supp.pdf

https://lan-lw.github.io/CIG/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a vision-language task utilizing queries comprising images and textual descriptions to achieve precise image retrieval. This task seeks to find images that are visually similar to a reference image while incorporating specific changes or features described textually (visual delta). CIR enables a more flexible and user-specific retrieval by bridging visual data with verbal instructions. This paper introduces a novel generative method that augments Composed Image Retrieval by Composed Image Generation (CIG) to provide pseudotarget images. CIG utilizes a textual inversion network to map reference images into semantic word space, which generates pseudo-target images in combination with textual descriptions. These images serve as additional visual information, significantly improving the accuracy and relevance of retrieved images when integrated into existing retrieval frameworks. Experiments conducted across multiple CIR datasets and several baseline methods demonstrate improvements in retrieval performance, which shows the potential of our approach as an effective add-on for existing composed image retrieval.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{xinLiu2025,

title = {Unveiling the Ignorance of MLLMs: Seeing Clearly, Answering Incorrectly},

author = {Ye xin Liu and Zhengyang Liang and Yueze Wang and Xianfeng Wu and Feilong Tang and Muyang He and Jian Li and Zheng Liu and Harry Yang and Ser-Nam Lim and Bo Zhao},

url = {https://arxiv.org/html/2406.10638v2

https://arxiv.org/pdf/2406.10638v2},

year  = {2025},

date = {2025-06-11},

abstract = {Multimodal Large Language Models (MLLMs) have displayed remarkable performance in multi-modal tasks, particularly in visual comprehension. However, we reveal that MLLMs often generate incorrect answers even when they understand the visual content. To this end, we manually construct a benchmark with 12 categories and design evaluation metrics that assess the degree of error in MLLM responses even when the visual content is seemingly understood. Based on this benchmark, we test 15 leading MLLMs and analyze the distribution of attention maps and logits of some MLLMs. Our investigation identifies two primary issues: 1) most instruction tuning datasets predominantly feature questions that “directly” relate to the visual content, leading to a bias in MLLMs’ responses to other indirect questions, and 2) MLLMs’ attention to visual tokens is notably lower than to system and question tokens. We further observe that attention scores between questions and visual tokens as well as the model’s confidence in the answers are lower in response to misleading questions than to straightforward ones. To address the first challenge, we introduce a paired positive and negative data construction pipeline to diversify the dataset. For the second challenge, we propose to enhance the model’s focus on visual content during decoding by refining the text and visual prompt. For the text prompt, we propose a content guided refinement strategy that performs preliminary visual content analysis to generate structured information before answering the question. Additionally, we employ a visual attention refinement strategy that highlights question-relevant visual tokens to increase the model’s attention to visual content that aligns with the question. Extensive experiments demonstrate that these challenges can be significantly mitigated with our proposed dataset and techniques. The benchmark, training set, and code will be available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ghosal2025,

title = {IMMUNELogo : Improving Safety Against Jailbreaks in Multi-modal LLMs via Inference-Time Alignment},

author = {Soumya Suvra Ghosal, Souradip Chakraborty, Vaibhav Singh, Tianrui Guan, Mengdi Wang, Ahmad Beirami, Furong Huang, Alvaro Velasquez, Dinesh Manocha, Amrit Singh Bedi},

url = {https://itsvaibhav01.github.io/immune-web/

https://arxiv.org/pdf/2411.18688

https://arxiv.org/abs/2411.18688

https://github.com/itsvaibhav01/Immune},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {With the widespread deployment of Multimodal Large Language Models (MLLMs) for visual-reasoning tasks, improving their safety has become crucial. Recent research indicates that despite training-time safety alignment, these models remain vulnerable to jailbreak attacks. In this work, we first highlight an important safety gap to describe that alignment achieved solely through safety training may be insufficient against jailbreak attacks. To address this vulnerability, we propose Immune, an inference-time defense framework that leverages a safe reward model through controlled decoding to defend against jailbreak attacks. Additionally, we provide a mathematical characterization of Immune, offering insights on why it improves safety against jailbreaks. Extensive evaluations on diverse jailbreak benchmarks using recent MLLMs reveal that Immune effectively enhances model safety while preserving the model's original capabilities. For instance, against text-based jailbreak attacks on LLaVA-1.6, Immune reduces the attack success rate by 57.82% and 16.78% compared to the base MLLM and state-of-the-art defense strategy, respectively.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025b,

title = {Enhancing Privacy-Utility Trade-offs to Mitigate Memorization in Diffusion Models},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://cvpr.thecvf.com/virtual/2025/poster/34842},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Text-to-image diffusion models have demonstrated remarkable capabilities in creating images highly aligned with user prompts, yet their proclivity for memorizing training set images has sparked concerns about the originality of the generated images and privacy issues, potentially leading to legal complications for both model owners and users, particularly when the memorized images contain proprietary content. Although methods to mitigate these issues have been suggested, enhancing privacy often results in a significant decrease in the utility of the outputs, as indicated by text-alignment scores. To bridge the research gap, we introduce a novel method, PRSS, which refines the classifier-free guidance approach in diffusion models by integrating prompt re-anchoring (PR) to improve privacy and incorporating semantic prompt search (SS) to enhance utility. Extensive experiments across various privacy levels demonstrate that our approach consistently improves the privacy-utility trade-off, establishing a new state-of-the-art.



},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Deng2025,

title = {Motion-Grounded Video Reasoning: Understanding and Perceiving Motion at Pixel Level},

author = {Andong Deng and Tongjia Chen and Shoubin Yu and Taojiannan Yang and Lincoln Spencer and Yapeng Tian and Ajmal Mian and Mohit Bansal and Chen Chen},

url = {https://groundmore.github.io/

https://arxiv.org/abs/2411.09921

https://github.com/dengandong/GroundMoRe

https://huggingface.co/datasets/groundmore/GroundMoRe},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we introduce Motion-Grounded Video Reasoning, a new motion understanding task that requires generating spatiotemporal segmentation masks according to the input question, and hence needs implicit spatiotemporal reasoning and grounding. This task extends existing spatiotemporal grounding work that focuses on explicit action/motion recognition, to a more general format by enabling implicit motion reasoning via questions. To facilitate the development of advanced motion-grounding models on such a task, we collect a large-scale dataset called GroundMoRe, which comprises 1,715 video clips, 249K object masks that are deliberately designed with 4 types (Causal, Sequential, Counterfactual, and Descriptive) for benchmarking deep and comprehensive motion understanding abilities. Our GroundMoRe uniquely requires models to generate visual answers (spatiotemporal masks), providing a more concrete and visually interpretable response than plain text. It evaluates models on spatiotemporal grounding and reasoning, helping address complex challenges in video reasoning, temporal perception, and pixel-level understanding. To further facilitate the proposed task, we propose a baseline model, Motion-Grounded Video Reasoning Assistant (MoRA). MoRA incorporates the multimodal reasoning ability from Multimodal LLM and the pixel-level perception capability from the grounding model (SAM) as well as an additional temporal localization head. MoRA achieves respectable performance on GroundMoRe outperforming the best existing visual grounding baseline model by an average of 28.8% relatively, but there still remains substantial room for interesting future improvements by the community. We hope this novel and challenging task will pave the way for future advancements in robust and general motion understanding via video reasoning segmentation.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ren2025,

title = {DA-VPT: Semantic-Guided Visual Prompt Tuning for Vision Transformers},

author = {Li Ren and Chen Chen and Liqiang Wang and Kien A. Hua},

url = {https://cvpr.thecvf.com/virtual/2025/poster/33078},

year  = {2025},

date = {2025-06-11},

abstract = {Visual Prompt Tuning (VPT) has become a promising solution for Parameter-Efficient Fine-Tuning (PEFT) approach for Vision Transformer (ViT) models by partially fine-tuning learnable tokens while keeping most model parameters frozen. Recent research has explored modifying the connection structures of the prompts. However, the fundamental correlation and distribution between the prompts and image tokens remain unexplored. In this paper, we leverage textit{metric learning} techniques to investigate how the distribution of prompts affects fine-tuning performance. Specifically, we propose a novel framework, textbf{D}istribution textbf{A}ware textbf{V}isual textbf{P}rompt Tuning (DA-VPT), to guide the distributions of the prompts by learning the distance metric from their class-related semantic data. Our method demonstrates that the prompts can serve as an effective bridge to share semantic information between image patches and the class token. We extensively evaluated our approach on popular benchmarks in both recognition and segmentation tasks. The results demonstrate that our approach enables more effective and efficient fine-tuning of ViT models by leveraging semantic information to guide the learning of the prompts, leading to improved performance on various downstream vision tasks. The code will be released.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Deng2025b,

title = {Seq2Time: Sequential Knowledge Transfer for Video LLM Temporal Grounding},

author = {Andong Deng and Zhongpai Gao and Anwesa Choudhuri and Benjamin Planche and Meng Zheng and Bin Wang and Terrence Chen and Chen Chen and Ziyan Wu},

url = {https://arxiv.org/abs/2411.16932},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Temporal awareness is essential for video large language models (LLMs) to understand and reason about events within long videos, enabling applications like dense video captioning and temporal video grounding in a unified system. However, the scarcity of long videos with detailed captions and precise temporal annotations limits their temporal awareness. In this paper, we propose Seq2Time, a data-oriented training paradigm that leverages sequences of images and short video clips to enhance temporal awareness in long videos. By converting sequence positions into temporal annotations, we transform large-scale image and clip captioning datasets into sequences that mimic the temporal structure of long videos, enabling self-supervised training with abundant time-sensitive data. To enable sequence-to-time knowledge transfer, we introduce a novel time representation that unifies positional information across image sequences, clip sequences, and long videos. Experiments demonstrate the effectiveness of our method, achieving a 27.6% improvement in F1 score and 44.8% in CIDEr on the YouCook2 benchmark and a 14.7% increase in recall on the Charades-STA benchmark compared to the baseline.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Intent3D: 3D Object Detection in RGB-D Scans Based on Human Intention},

author = {Weitai Kang and Mengxue Qu and Jyoti Kini and Yunchao Wei and Mubarak Shah and Yan Yan},

url = {https://openreview.net/forum?id=5GgjiRzYp3&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/3450_Intent3D_3D_Object_Detect.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In real-life scenarios, humans seek out objects in the 3D world to fulfill their daily needs or intentions. This inspires us to introduce 3D intention grounding, a new task in 3D object detection employing RGB-D, based on human intention, such as “I want something to support my back.” Closely related, 3D visual grounding focuses on understanding human reference. To achieve detection based on human intention, it relies on humans to observe the scene, reason out the target that aligns with their intention (“pillow” in this case), and finally provide a reference to the AI system, such as “A pillow on the couch”. Instead, 3D intention grounding challenges AI agents to automatically observe, reason and detect the desired target solely based on human intention. To tackle this challenge, we introduce the new Intent3D dataset, consisting of 44,990 intention texts associated with 209 f ine-grained classes from 1,042 scenes of the ScanNet [Dai et al., 2017] dataset. We also establish several baselines based on different language-based 3D object detection models on our benchmark. Finally, we propose IntentNet, our unique approach, designed to tackle this intention-based detection problem. It focuses on three key aspects: intention understanding, reasoning to identify object candidates, and cascaded adaptive learning that leverages the intrinsic priority logic of different losses for multiple objective optimization.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chhipa2025,

title = {ASTrA: Adversarial Self-supervised Training with Adaptive-Attacks},

author = {Prakash Chandra Chhipa and Gautam Vashishtha and Settur Jithamanyu and Rajkumar Saini and Mubarak Shah and Marcus Liwicki},

url = {https://prakashchhipa.github.io/projects/ASTrA

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7338_ASTrA_Adversarial_Self_su.pdf.pdf},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Existing self-supervised adversarial training (self-AT) methods rely on handcrafted adversarial attack strategies for PGD attacks, which fail to adapt to the evolving learning dynamics of the model and do not account for instancespecific characteristics of images. This results in sub-optimal adversarial robustness and limits the alignment between clean and adversarial data distributions. To address this, we propose ASTrA (Adversarial Self-supervised Training with Adaptive-Attacks), a novel framework introducing a learnable, self-supervised attack strategy network that autonomously discovers optimal attack parameters through exploration-exploitation in a single training episode. ASTrA leverages a reward mechanism based on contrastive loss, optimized with REINFORCE, enabling adaptive attack strategies without labeled data or additional hyperparameters. We further introduce a mixed contrastive objective to align the distribution of clean and adversarial examples in representation space. ASTrA achieves state-of-the-art results on CIFAR10, CIFAR100, and STL10 while integrating seamlessly as a plug-and-play module for other self-AT methods. ASTrAshows scalability to larger datasets, demonstrates strong semi-supervised performance, and is resilient to robust overfitting, backed by explainability analysis on optimal attack strategies. Project page for source code and other details at https://prakashchhipa.github.io/projects/ASTrA.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2025,

title = {ALBAR: Adversarial Learning approach to mitigate Biases in Action Recognition},

author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://openreview.net/forum?id=9KiE3t6CsL&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7609_ALBAR_Adversarial_Learnin.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Bias in machine learning models can lead to unfair decision making, and while it has been well-studied in the image and text domains, it remains underexplored in action recognition. Action recognition models often suffer from background bias (i.e., inferring actions based on background cues) and foreground bias (i.e., relying on subject appearance), which can be detrimental to real-life applications such as autonomous vehicles or assisted living monitoring. While prior approaches have mainly focused on mitigating background bias using specialized augmentations, we thoroughly study both biases. We propose approachname, a novel adversarial training method that mitigates foreground and background biases without requiring specialized knowledge of the bias attributes. Our framework applies an adversarial cross-entropy loss to the sampled static clip (where all the frames are the same) and aims to make its class probabilities uniform using a proposed textit{entropy maximization} loss. Additionally, we introduce a textit{gradient penalty} loss for regularization against the debiasing process. We evaluate our method on established background and foreground bias protocols, setting a new state-of-the-art and strongly improving combined debiasing performance by over textbf{12%} on HMDB51. Furthermore, we identify an issue of background leakage in the existing UCF101 protocol for bias evaluation which provides a shortcut to predict actions and does not provide an accurate measure of the debiasing capability of a model. We address this issue by proposing more fine-grained segmentation boundaries for the actor, where our method also outperforms existing approaches.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2025,

title = {AdaIR: Adaptive All-in-One Image Restoration via Frequency Mining and Modulation},

author = {Yuning Cui and Syed Waqas Zamir and Salman Khan and Alois Knoll and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://openreview.net/forum?id=M5t0WvjfCg&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/8311_AdaIR_Adaptive_All_in_One.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In the image acquisition process, various forms of degradation, including noise, blur, haze, and rain, are frequently introduced. These degradations typically arise from the inherent limitations of cameras or unfavorable ambient conditions. To recover clean images from their degraded versions, numerous specialized restoration methods have been developed, each targeting a specific type of degradation. Recently, all-in-one algorithms have garnered significant attention by addressing different types of degradations within a single model without requiring the prior information of the input degradation type. However, most methods purely operate in the spatial domain and do not delve into the distinct frequency variations inherent to different degradation types. To address this gap, we propose an adaptive all-in-one image restoration network based on frequency mining and modulation. Our approach is motivated by the observation that different degradation types impact the image content on different frequency subbands, thereby requiring different treatments for each restoration task. Specifically, we first mine low- and high-frequency information from the input features, guided by the adaptively decoupled spectra of the degraded image. The extracted features are then modulated by a bidirectional operator to facilitate interactions between different frequency components. Finally, the modulated features are merged into the original input for a progressively guided restoration. With this approach, the model achieves adaptive reconstruction by accentuating the informative frequency subbands according to different input degradations. Extensive experiments demonstrate that the proposed method, named AdaIR, achieves state-of-the-art performance on different image restoration tasks, including image denoising, dehazing, deraining, motion deblurring, and low-light image enhancement. Our code and models will be made publicly available.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025,

title = {Exploring Local Memorization in Diffusion Models via Bright Ending Attention},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://openreview.net/forum?id=p4cLtzk4oe&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/9283_Exploring_Local_Memorizat.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In this paper, we identify and leverage a novel `bright ending' (BE) anomaly in diffusion models prone to memorizing training images to address a new task: locating localized memorization regions within these models. BE refers to a distinct cross-attention pattern observed in text-to-image generations using diffusion models. Specifically, memorized image patches exhibit significantly greater attention to the end token during the final inference step compared to non-memorized patches. This attention map effectively highlights regions where the generated image replicates training data. Furthermore, driven by our observation that local memorization significantly underperforms in existing tasks of measuring, detecting, and mitigating memorization in diffusion models compared to global memorization, we propose a simple yet effective method to integrate BE and the results of the new localization task into these existing frameworks. This integration effectively improves their performances by narrowing the performance gap caused by local memorization. Our results not only demonstrate the successful execution of the new localization task but also establish new state-of-the-art performance across all existing tasks, underscoring the significance of the BE phenomenon.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Monsefi2025,

title = {Frequency-Guided Masking for Enhanced Vision Self-Supervised Learning },

author = {Amin Karimi Monsefi and Mengxi Zhou and Nastaran Karimi Monsefi and Ser-Nam Lim and Wei-Lun Chao and Rajiv Ramnath},

url = {https://arxiv.org/abs/2409.10362

https://arxiv.org/pdf/2409.10362},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {We present a novel frequency-based Self-Supervised Learning (SSL) approach that significantly enhances its efficacy for pre-training. Prior work in this direction masks out pre-defined frequencies in the input image and employs a reconstruction loss to pre-train the model. While achieving promising results, such an implementation has two fundamental limitations as identified in our paper. First, using pre-defined frequencies overlooks the variability of image frequency responses. Second, pre-trained with frequency-filtered images, the resulting model needs relatively more data to adapt to naturally looking images during fine-tuning. To address these drawbacks, we propose FOurier transform compression with seLf-Knowledge distillation (FOLK), integrating two dedicated ideas. First, inspired by image compression, we adaptively select the masked-out frequencies based on image frequency responses, creating more suitable SSL tasks for pre-training. Second, we employ a two-branch framework empowered by knowledge distillation, enabling the model to take both the filtered and original images as input, largely reducing the burden of downstream tasks. Our experimental results demonstrate the effectiveness of FOLK in achieving competitive performance to many state-of-the-art SSL methods across various downstream tasks, including image classification, few-shot learning, and semantic segmentation.

},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2025,

title = {Contextual Self-paced Learning for Weakly Supervised Spatio-Temporal Video Grounding},

author = {Akash Kumar and Zsolt Kira and Yogesh Rawat},

url = {https://arxiv.org/pdf/2501.17053

https://github.com/AKASH2907/copsal-weakly-stvg

https://akash2907.github.io/cospal_webpage/

},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In this work, we focus on Weakly Supervised Spatio-Temporal Video Grounding (WSTVG). It is a multimodal task aimed at localizing specific subjects spatiotemporally based on textual queries without bounding box supervision. Motivated by recent advancements in multi-modal foundation models for grounding tasks, we f irst explore the potential of state-of-the-art object detection models for WSTVG. Despite their robust zero-shot capabilities, our adaptation reveals significant limitations, including inconsistent temporal predictions, inadequate understanding of complex queries, and challenges in adapting to difficult scenarios. We propose CoSPaL (Contextual Self-Paced Learning), a novel approach which is designed to overcome these limitations. CoSPaL integrates three core components: (1) Tubelet Phrase Grounding (TPG), which introduces spatio-temporal prediction by linking textual queries to tubelets; (2) Contextual Referral Grounding (CRG), which improves comprehension of complex queries by extracting contextual information to refine object identification over time; and (3) Self-Paced Scene Understanding (SPS), a training paradigm that progressively increases task difficulty, enabling the model to adapt to complex scenarios by transitioning from coarse to fine-grained understanding. We demonstrate the effectiveness of CoSPaL on three benchmark WSTVGdatasets, achieving a 3.9% absolute improvement on VidSTG and a 7.9% improvement on HCSTVG-v1.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tang2025,

title = {Intervening Anchor Token: Decoding Strategy in Alleviating Hallucinations for MLLMs},

author = {Feilong Tang and Zile Huang and Chengzhi Liu and Qiang Sun and Harry Yang and Ser-Nam Lim},

url = {https://openreview.net/forum?id=zGb4WgCW5i

https://openreview.net/pdf?id=zGb4WgCW5i},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Multimodal large language models (MLLMs) offer a powerful mechanism for interpreting visual information. However, they often suffer from hallucinations, which impede the real-world usage of these models. Existing methods attempt to alleviate this issue by designing special decoding strategies that penalize the summary tokens. However, these methods lack analysis of the relationship between hallucination and summarization mechanism of LLMs. Interestingly, we find that penalizing summary tokens is not necessary: merely intervening the query-key parameters variance, without costing extra inference time, still alleviates hallucinations. Specifically, we explore the causes of hallucinations by analyzing localized self-attention patterns called ``anchor" tokens and define the attention localization degree of the model as token propagation probabilities. Our analysis reveals that over-propagation of anchor tokens occurs when the distribution of eigenvalues of the query and key matrices has a non-zero mean and a polarized variance, leading to excessive dependence on anchor tokens while neglecting vision information and describes the image content with hallucination. Based on the observation, we propose a versatile plug-and-play decoding strategy, Dynamic Token Propagation Mechanism (TAME), to alleviate excessive propagation by dynamically intervening the eigenspectrum variance of the attention weight, thereby alleviating hallucinations without relying on complex decoding strategies. Extensive experiments reveal a correlation between the eigenspectrum and hallucinations across various MLLMs, and show that TAME reduces the percentage of hallucinated objects.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huang2025,

title = {LASER: A Neuro-Symbolic Framework for Learning Spatio-Temporal Scene Graphs with Weak Supervision},

author = {Jiani Huang and Ziyang Li and Mayur Nail and Ser-Nam Lim},

url = {https://arxiv.org/abs/2304.07647

https://arxiv.org/pdf/2304.07647},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {We propose LASER, a neuro-symbolic approach to learn semantic video representations that capture rich spatial and temporal properties in video data by leveraging high-level logic specifications. In particular, we formulate the problem in terms of alignment between raw videos and spatio-temporal logic specifications. The alignment algorithm leverages a differentiable symbolic reasoner and a combination of contrastive, temporal, and semantics losses. It effectively and efficiently trains low-level perception models to extract a fine-grained video representation in the form of a spatio-temporal scene graph that conforms to the desired high-level specification. To practically reduce the manual effort of obtaining ground truth labels, we derive logic specifications from captions by employing a large language model with a generic prompting template. In doing so, we explore a novel methodology that weakly supervises the learning of spatio-temporal scene graphs with widely accessible video-caption data. We evaluate our method on three datasets with rich spatial and temporal specifications: 20BN-Something-Something, MUGEN, and OpenPVSG. We demonstrate that our method learns better fine-grained video semantics than existing baselines.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pathak2025,

title = {LR0.FM: Low-Resolution Zero-Shot Classification Benchmark for Foundation Models},

author = {Priyank Pathak and Shyam Marjit and Shruti Vyas and Yogesh Rawat},

url = {https://arxiv.org/pdf/2502.03950

https://ucf-crcv.github.io/lr0.fm/

https://github.com/shyammarjit/LR0.FM

},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Visual-language foundation Models (FMs) exhibit remarkable zero-shot generalization across diverse tasks, largely attributed to extensive pre-training on largescale datasets. However, their robustness on low-resolution/pixelated (LR) images, a common challenge in real-world scenarios, remains underexplored. We introduce this http URL, a comprehensive benchmark evaluating the impact of low resolution on the zero-shot classification performance of 10 FM(s) across 66 backbones and 15 datasets. We propose a novel metric, Weighted Aggregated Robustness, to address the limitations of existing metrics and better evaluate model performance across resolutions and datasets. Our key findings show that: (i) model size positively correlates with robustness to resolution degradation, (ii) pre-training dataset quality is more important than its size, and (iii) fine-tuned and higher resolution models are less robust against LR. Our analysis further reveals that the model makes semantically reasonable predictions at LR, and the lack of fine-grained details in input adversely impacts the model's initial layers more than the deeper layers. We use these insights and introduce a simple strategy, LR-TK0, to enhance the robustness of models without compromising their pre-trained weights. We demonstrate the effectiveness of LR-TK0 for robustness against low-resolution across several datasets and its generalization capability across backbones and other approaches. },

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhang2025,

title = {Model-Free Offline Reinforcement Learning with Enhanced Robustness},

author = {Chi Zhang and Zain Ulabedeen Farhat and George K. Atia and Yue Wang},

url = {https://openreview.net/forum?id=QyVLJ7EnAC

https://openreview.net/pdf?id=QyVLJ7EnAC},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Offline reinforcement learning (RL) has gained considerable attention for its ability to learn policies from pre-collected data without real-time interaction, which makes it particularly useful for high-risk applications. However, due to its reliance on offline datasets, existing works inevitably introduce assumptions to ensure effective learning, which, however, often lead to a trade-off between robustness to model mismatch and scalability to large environments. In this paper, we enhance both aspects with a novel double-pessimism principle, which conservatively estimates performance and accounts for both limited data and potential model mismatches, two major reasons for the previous trade-off. We then propose a universal, model-free algorithm to learn an optimal policy that is robust to potential environment mismatches, which enhances robustness in a scalable manner. Furthermore, we provide a sample complexity analysis of our algorithm when the mismatch is modeled by the 

-norm, which also theoretically demonstrates the efficiency of our method. Extensive experiments further demonstrate that our approach significantly improves robustness in a more scalable manner than existing methods.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {GOttack: Universal Adversarial Attacks on Graph Neural Networks via Graph Orbits Learning},

author = {Zulfikar Alom and Tran Gia Bao Ngo and Murat Kantarcioglu and Cuneyt Gurcan Akcora},

url = {https://openreview.net/forum?id=YbURbViE7l

https://openreview.net/pdf?id=YbURbViE7l

https://github.com/cakcora/GOttack},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

publisher = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Graph Neural Networks (GNNs) have demonstrated superior performance in node classification tasks across diverse applications. However, their vulnerability to adversarial attacks, where minor perturbations can mislead model predictions, poses significant challenges. This study introduces GOttack, a novel adversarial attack framework that exploits the topological structure of graphs to undermine the integrity of GNN predictions systematically.



By defining a topology-aware method to manipulate graph orbits, our approach generates adversarial modifications that are both subtle and effective, posing a severe test to the robustness of GNNs. We evaluate the efficacy of GOttack across multiple prominent GNN architectures using standard benchmark datasets. Our results show that GOttack outperforms existing state-of-the-art adversarial techniques and completes training in approximately 55% of the time required by the fastest competing model, achieving the highest average misclassification rate in 155 tasks. This work not only sheds light on the susceptibility of GNNs to structured adversarial attacks but also shows that certain topological patterns may play a significant role in the underlying robustness of the GNNs. Our Python implementation is shared at https://github.com/cakcora/GOttack.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2025,

title = {3D Vision-Language Gaussian Splatting},

author = {Qucheng Peng and Benjamin Planche and Zhongpai Gao and Meng Zheng and Anwesa Choudhuri and Terrence Chen and Chen Chen and Ziyan Wu},

url = {https://iclr.cc/virtual/2025/poster/29604},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-04},

publisher = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Recent advancements in 3D reconstruction methods and vision-language models have propelled the development of multi-modal 3D scene understanding, which has vital applications in robotics, autonomous driving, and virtual/augmented reality. However, current multi-modal scene understanding approaches have naively embedded semantic representations into 3D reconstruction methods without striking a balance between visual and language modalities, which leads to unsatisfying semantic rasterization of translucent or reflective objects, as well as over-fitting on color modality. To alleviate these limitations, we propose a solution that adequately handles the distinct visual and semantic modalities, i.e., a 3D vision-language Gaussian splatting model for scene understanding, to put emphasis on the representation learning of language modality. We propose a novel cross-modal rasterizer, using modality fusion along with a smoothed semantic indicator for enhancing semantic rasterization. We also employ a camera-view blending technique to improve semantic consistency between existing and synthesized views, thereby effectively mitigating over-fitting. Extensive experiments demonstrate that our method achieves state-of-the-art performance in open-vocabulary semantic segmentation, surpassing existing methods by a significant margin.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lai2025,

title = {Revisit Large-Scale Image-Caption Data in Pre-training Multimodal Foundation Models},

author = {Zhengfeng Lai and Vasileios Saveris and Chen Chen and Hong-You Chen and Haotian Zhang and Bowen Zhang and Wenze Hu and Juan Tebar and Zhe Gan and Peter Grasch and Meng Cao and Yinfei Yang},

url = {https://iclr.cc/virtual/2025/poster/29536},

year  = {2025},

date = {2025-04-24},

publisher = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Recent advancements in multimodal models highlight the value of rewritten captions for improving performance, yet key challenges remain. For example, while synthetic captions often provide superior quality and image-text alignment, it is not clear whether they can fully replace AltTexts: the role of synthetic captions and their interaction with original web-crawled AltTexts in pre-training is still not well understood. Moreover, different multimodal foundation models may have unique preferences for specific caption formats, but efforts to identify the optimal captions for each model remain limited. In this work, we propose a novel, controllable, and scalable captioning pipeline designed to generate diverse caption formats tailored to various multimodal models. By examining short synthetic captions (SSC) and descriptive synthetic captions (DSC) as case studies, we systematically explore their effects and interactions with AltTexts across models such as CLIP, multimodal LLMs, and diffusion models. Our findings reveal that a hybrid approach that keeps both synthetic captions and AltTexts can outperform the use of synthetic captions alone, improving both alignment and performance, with each model demonstrating preferences for particular caption formats. This comprehensive analysis provides valuable insights into optimizing captioning strategies, thereby advancing the pre-training of multimodal foundation models.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhang2025b,

title = {CipherPrune: Efficient and Scalable Private Transformer Inference},

author = {Yancheng Zhang and Jiaqi Xue and Mengxin Zheng and Mimi Xie and Mingzhe Zhang and Lei Jiang and Qian Lou},

url = { https://openreview.net/pdf?id=mUMvr33FTu

https://github.com/UCF-Lou-Lab-PET/cipher-prune-inference},

year  = {2025},

date = {2025-04-24},

publisher = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Private Transformer inference using cryptographic protocols offers promising solutions for privacy-preserving machine learning; however, it still faces significant runtime overhead (efficiency issues) and challenges in handling long-token inputs (scalability issues). We observe that the Transformer’s operational complexity scales quadratically with the number of input tokens, making it essential to reduce the input token length. Notably, each token varies in importance, and many inputs contain redundant tokens. Additionally, prior private inference methods that rely on high-degree polynomial approximations for non-linear activations are computationally expensive. Therefore, reducing the polynomial degree for less important tokens can significantly accelerate private inference. Building on these observations, we propose CipherPrune, an efficient and scalable private inference framework that includes a secure encrypted token pruning protocol, a polynomial reduction protocol, and corresponding Transformer network optimizations. At the protocol level, encrypted token pruning adaptively removes unimportant tokens from encrypted inputs in a progressive, layer-wise manner. Additionally, encrypted polynomial reduction assigns lower-degree polynomials to less important tokens after pruning, enhancing efficiency without decryption. At the network level, we introduce protocol-aware network optimization via a gradient-based search to maximize pruning thresholds and polynomial reduction conditions while maintaining the desired accuracy. Our experiments demonstrate that CipherPrune reduces the execution overhead of private Transformer inference by approximately 6.1× for 128-token inputs and 10.6× for 512-token inputs, compared to previous methods, with only a marginal drop in accuracy. The code is publicly available at https://github.com/UCF-Lou-Lab-PET/cipher-prune-inference.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wu2024,

title = {PTQ4DiT: Post-training Quantization for Diffusion Transformers},

author = {Junyi Wu and Haoxuan Wang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://nips.cc/virtual/2024/poster/95445

https://arxiv.org/pdf/2405.16005

https://github.com/adreamwu/PTQ4DiT},

year  = {2024},

date = {2024-12-13},

urldate = {2024-12-13},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The recent introduction of Diffusion Transformers (DiTs) has demonstrated exceptional capabilities in image generation by using a different backbone architecture, departing from traditional U-Nets and embracing the scalable nature of transformers. Despite their advanced capabilities, the wide deployment of DiTs, particularly for real-time applications, is currently hampered by considerable computational demands at the inference stage. Post-training Quantization (PTQ) has emerged as a fast and data-efficient solution that can significantly reduce computation and memory footprint by using low-bit weights and activations. However, its applicability to DiTs has not yet been explored and faces non-trivial difficulties due to the unique design of DiTs. In this paper, we propose PTQ4DiT, a specifically designed PTQ method for DiTs. We discover two primary quantization challenges inherent in DiTs, notably the presence of salient channels with extreme magnitudes and the temporal variability in distributions of salient activation over multiple timesteps. To tackle these challenges, we propose Channel-wise Salience Balancing (CSB) and Spearmen's 

ρ

-guided Salience Calibration (SSC). CSB leverages the complementarity property of channel magnitudes to redistribute the extremes, alleviating quantization errors for both activations and weights. SSC extends this approach by dynamically adjusting the balanced salience to capture the temporal variations in activation. Additionally, to eliminate extra computational costs caused by PTQ4DiT during inference, we design an offline re-parameterization strategy for DiTs. Experiments demonstrate that our PTQ4DiT successfully quantizes DiTs to 8-bit precision (W8A8) while preserving comparable generation ability and further enables effective quantization to 4-bit weight precision (W4A8) for the first time.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2024,

title = {Asynchronous Perception Machine for Test Time Training},

author = {Rajat Modi and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/apm_final.pdf

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-11},

urldate = {2024-12-11},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {In this work, we propose Asynchronous Perception Machine (APM), a computationally-efficient architecture for test-time-training (TTT). APM can process patches of an image one at a time in any order asymmetrically and still encode semantic-awareness. We demonstrate APM's ability to recognize out-of-distribution images without dataset-specific pre-training, and its competitive classification performance over existing TTT approaches. To perform TTT, APM just distills test sample's representation once. APM possesses a unique property: it can learn using just this single representation and starts predicting semantically-aware features. We demonstrate APM's potential application beyond test-time-training: APM can scale up to a dataset of 2D images and yield semantic-clusterings in a single forward pass. APM also provides first empirical evidence of GLOM's insight, i.e. percept is really a field. Therefore, APM helps us converge towards an implementation which can do both interpolation and perception on a shared-connectionist hardware. Our codebase has been provided for review and will be made publicly-available. --------It now appears that some of the ideas in GLOM could be made to work.https://www.technologyreview.com/2021/04/16/1021871/geoffrey-hinton-glom-godfather-ai-neural-networks/},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lim2024,

title = {AirSketch: Generative Motion to Sketch},

author = {Hui Xian Grace Lim and Xuanming Cui and Yogesh Singh Rawat and Ser-Nam Lim },

url = {https://arxiv.org/pdf/2407.08906

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-09},

urldate = {2024-07-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Illustration is a fundamental mode of human expression and communication. Certain types of motion that accompany speech can provide this illustrative mode of communication. While Augmented and Virtual Reality technologies (AR/VR) have introduced tools for producing drawings with hand motions (air drawing), they typically require costly hardware and additional digital markers, thereby limiting their accessibility and portability. Furthermore, air drawing demands considerable skill to achieve aesthetic results. To address these challenges, we introduce the concept of AirSketch, aimed at generating faithful and visually coherent sketches directly from hand motions, eliminating the need for complicated headsets or markers. We devise a simple augmentation-based self-supervised training procedure, enabling a controllable image diffusion model to learn to translate from highly noisy hand tracking images to clean, aesthetically pleasing sketches, while preserving the essential visual cues from the original tracking data. We present two air drawing datasets to study this problem. Our findings demonstrate that beyond producing photo-realistic images from precise spatial inputs, controllable image diffusion can effectively produce a refined, clear sketch from a noisy input. Our work serves as an initial step towards marker-less air drawing and reveals distinct applications of controllable diffusion models to AirSketch and AR/VR in general.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Morafah2024,

title = {Towards Diverse Device Heterogeneous Federated Learning via Task Arithmetic Knowledge Integration},

author = {Mahdi Morafah and Vyacheslav Kungurtsev and Hojin Matthew Chang and Chen Chen and Bill Lin},

url = {https://arxiv.org/pdf/2409.18461

https://mmorafah.github.io/takflpage/

https://github.com/MMorafah/TAKFL},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated Learning (FL) has emerged as a promising paradigm for collaborative machine learning, while preserving user data privacy. Despite its potential, standard FL algorithms lack support for diverse heterogeneous device prototypes, which vary significantly in model and dataset sizes—from small IoT devices to large workstations. This limitation is only partially addressed by existing knowledge distillation (KD) techniques, which often fail to transfer knowledge effectively across a broad spectrum of device prototypes with varied capabilities. This failure primarily stems from two issues: the dilution of informative logits from more capable devices by those from less capable ones, and the use of a single integrated logits as the distillation target across all devices, which neglects their individual learning capacities and and the unique contributions of each device. To address these challenges, we introduce TAKFL, a novel KD-based framework that treats the knowledge transfer from each device prototype’s ensemble as a separate task, independently distilling each to preserve its unique contributions and avoid dilution. TAKFL also incorporates a KD-based self-regularization technique to mitigate the issues related to the noisy and unsupervised ensemble distillation process. To integrate the separately distilled knowledge, we introduce an adaptive task arithmetic knowledge integration process, allowing each student model to customize the knowledge integration for optimal performance. Additionally, we present theoretical results demonstrating the effectiveness of task arithmetic in transferring knowledge across heterogeneous device prototypes with varying capacities. Comprehensive evaluations of our method across both computer vision (CV) and natural language processing (NLP) tasks demonstrate that TAKFL achieves state-of-the-art results in a variety of datasets and settings, significantly outperforming existing KD-based methods. Our code is released at https://github.com/MMorafah/TAKFL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024c,

title = {Taming Cross-Domain Representation Variance in Federated Prototype Learning with Heterogeneous Data Domains},

author = {Lei Wang and Jieming Bian and Letian Zhang and Chen Chen and Jie Xu},

url = {https://arxiv.org/pdf/2403.09048

https://arxiv.org/abs/2403.09048},

year  = {2024},

date = {2024-12-09},

urldate = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated learning (FL) allows collaborative machine learning training without sharing private data. While most FL methods assume identical data domains across clients, real-world scenarios often involve heterogeneous data domains. Federated Prototype Learning (FedPL) addresses this issue, using mean feature vectors as prototypes to enhance model generalization. However, existing FedPL methods create the same number of prototypes for each client, leading to cross-domain performance gaps and disparities for clients with varied data distributions. To mitigate cross-domain feature representation variance, we introduce FedPLVM, which establishes variance-aware dual-level prototypes clustering and employs a novel α-sparsity prototype loss. The dual-level prototypes clustering strategy creates local clustered prototypes based on private data features, then performs global prototypes clustering to reduce communication complexity and preserve local data privacy. The α-sparsity prototype loss aligns samples from underrepresented domains, enhancing intra-class similarity and reducing inter-class similarity. Evaluations on Digit-5, Office-10, and DomainNet datasets demonstrate our method's superiority over existing approaches.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2024,

title = {Advancing Video Anomaly Detection: A Concise Review and a New Dataset},

author = {Liyun Zhu, Lei Wang, Arjun Raj, Tom Gedeon, Chen Chen},

url = {https://arxiv.org/pdf/2402.04857

https://arxiv.org/abs/2402.04857

https://msad-dataset.github.io/},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Video Anomaly Detection (VAD) finds widespread applications in security surveillance, traffic monitoring, industrial monitoring, and healthcare. Despite extensive research efforts, there remains a lack of concise reviews that provide insightful guidance for researchers. Such reviews would serve as quick references to grasp current challenges, research trends, and future directions. In this paper, we present such a review, examining models and datasets from various perspectives. We emphasize the critical relationship between model and dataset, where the quality and diversity of datasets profoundly influence model performance, and dataset development adapts to the evolving needs of emerging approaches. Our review identifies practical issues, including the absence of comprehensive datasets with diverse scenarios. To address this, we introduce a new dataset, Multi-Scenario Anomaly Detection (MSAD), comprising 14 distinct scenarios captured from various camera views. Our dataset has diverse motion patterns and challenging variations, such as different lighting and weather conditions, providing a robust foundation for training superior models. We conduct an in-depth analysis of recent representative models using MSAD and highlight its potential in addressing the challenges of detecting anomalies across diverse and evolving surveillance scenarios. Our dataset is available here.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Arul2024,

title = {When, What, and with Whom to Communicate: Enhancing RL-based Multi-Robot Navigation through Selective Communication},

author = {S. H. Arul and A. S. Bedi and D. Manocha},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shek2024,

title = {LANCAR: Leveraging Language for Context-Aware Robot Locomotion in Unstructured Environments},

author = {C. L. Shek and X. Wu and W. A. Suttle and C. Busart and E. Zaroukian and D. Manocha and P. Tokekar and A. S. Bedi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2310.00481v2.pdf},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024b,

title = {TrustNavGPT: Trust-Driven Audio-Guided Robot Navigation under Uncertainty with Large Language Models},

author = {Xingpeng Sun and Yiran Zhang and Xindi Tang and Amrit Singh Bedi and Aniket Bera},

url = {https://xingpengsun0.github.io/trustnav/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS) [Oral]},

abstract = {Large language models (LLMs) exhibit a wide range of promising capabilities -- from step-by-step planning to commonsense reasoning --that provide utility for robot navigation. However, as humans communicate with robots in the real world, ambiguity and uncertainty may be embedded inside spoken instructions. While LLMs are proficient at processing text in human conversations, they often encounter difficulties with the nuances of verbal instructions and, thus, remain prone to hallucinate trust in human command. In this work, we present TrustNavGPT, an LLM-based audio-guided navigation agent that uses affective cues in spoken communication—elements such as tone and inflection that convey meaning beyond words—allowing it to assess the trustworthiness of human commands and make effective, safe decisions.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {GDM-Net: gas distribution mapping with a mobile robot using deep reinforcement learning and gaussian process regression},

author = {Iliya Kulbaka and Ayan Dutta and O. Patrick Kreidl and Ladislau Bölöni and Swapnoneel Roy},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2024,

title = {Sparse Points to Dense Clouds: Enhancing 3D Detection with Limited LiDAR Data},

author = {Aakash Kumar and Chen Chen and Ajmal Mian and Neils Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2404.06715v1.pdf

https://aakashjuseja-aj.github.io/Sparse_to_Dense/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

abstract = {3D detection is a critical task that enables machines to identify and locate objects in three-dimensional space. It has a broad range of applications in several fields, including autonomous driving, robotics and augmented reality. Monocular 3D detection is attractive as it requires only a single camera, however, it lacks the accuracy and robustness required for real world applications. High resolution LiDAR on the other hand, can be expensive and lead to interference problems in heavy traffic given their active transmissions. We propose a balanced approach that combines the advantages of monocular and point cloud-based 3D detection. Our method requires only a small number of 3D points, that can be obtained from a low-cost, low-resolution sensor. Specifically, we use only 512 points, which is just 1% of a full LiDAR frame in the KITTI dataset. Our method reconstructs a complete 3D point cloud from this limited 3D information combined with a single image. The reconstructed 3D point cloud and corresponding image can be used by any multi-modal off-the-shelf detector for 3D object detection. By using the proposed network architecture with an off-the-shelf multi-modal 3D detector, the accuracy of 3D detection improves by 20% compared to the state-of-the-art monocular detection methods and 6% to 9% compare to the baseline multi-modal methods on KITTI and JackRabbot datasets.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar

Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf

https://prakashchhipa.github.io/projects/mpd

https://youtu.be/MKh9NE_XEMY

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_presentation_ECCV2024_final.pdf},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method  outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200

},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf

https://daveishan.github.io/finepsuedo-webpage/

https://youtu.be/bWOd8_JpjQs?si=WWRDhdg5ADWL0uwB},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias,  classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48,  FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the  robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf

https://arxiv.org/abs/2407.13851

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.

},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and benjamin Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024b,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf

https://daveishan.github.io/avr-webpage/

https://youtu.be/6euQwz7XdQk?si=v12dQH4e7UrTUrIU},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf

https://arxiv.org/pdf/2407.04370

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tai2024,

title = {uCAP: An Unsupervised Prompting Method for Vision-Language Models},

author = {Kai Sheng Tai and Sirius Chen and Satya Narayan Shukla and Hanchao Yu and Philip Torr and Taipeng Tian and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses a significant limitation that prevents Contrastive Language-Image Pretrained Models (CLIP) from achieving optimal performance on downstream image classification tasks. The key problem with CLIP-style zero-shot classification is that it requires domain-specific context in the form of prompts to better align the class descriptions to the downstream data distribution. In particular, prompts for vision-language models are domain-level texts (e.g., “a centered satellite image of ...”) which, together with the class names, are fed into the text encoder to provide more context for the downstream dataset. These prompts are typically manually tuned, which is time consuming and often sub-optimal. To overcome this bottleneck, this paper proposes uCAP, a method to automatically learn domain-specific prompts/contexts using only unlabeled in-domain images. We achieve this by modeling the generation of images given the class names and a domain-specific prompt with an unsupervised likelihood distribution, and then performing inference of the prompts. We validate the proposed method across various models and datasets, showing that uCAP consistently outperforms manually tuned prompts and related baselines on the evaluated datasets: ImageNet, CIFAR-10, CIFAR-100, OxfordPets (up to 2%), SUN397 (up to 5%), and Caltech101 (up to 3%).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024b,

title = {Fast Encoding and Decoding for Implicit Video Representation},

author = {Hao Chen and Saining Xie and Ser-Nam Lim and Abhinav Shrivastava},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jang2024,

title = {Spherical Linear Interpolation and Text-Anchoring for Zero-shot Composed Retrieval},

author = {Young Kyun Jang and Dat Huynh and Ashish Shah and Wen-Kai Chen and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024,

title = {Towards Multi-modal Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Aritra Dutta and Xin Li and Chen Chen},

url = {https://arxiv.org/pdf/2404.12467.pdf

https://github.com/imguangyu/FedCola},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2024,

title = {ControlNet++: Improving Conditional Controls with Efficient Consistency Feedback},

author = {Ming Li and Taojiannan Yang and Huafeng Kuang and Jie Wu and Zhaoning Wang and Xuefeng Xiao and Chen Chen},

url = {https://arxiv.org/pdf/2404.07987.pdf

https://liming-ai.github.io/ControlNet_Plus_Plus/

https://github.com/liming-ai/ControlNet_Plus_Plus

https://huggingface.co/spaces/limingcv/ControlNet-Plus-Plus},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {To enhance the controllability of text-to-image diffusion models, existing efforts like ControlNet incorporated image-based conditional controls. In this paper, we reveal that existing methods still face significant challenges in generating images that align with the image conditional controls. To this end, we propose ControlNet++, a novel approach that improves controllable generation by explicitly optimizing pixel-level cycle consistency between generated images and conditional controls. Specifically, for an input conditional control, we use a pre-trained discriminative reward model to extract the corresponding condition of the generated images, and then optimize the consistency loss between the input conditional control and extracted condition. A straightforward implementation would be generating images from random noises and then calculating the consistency loss, but such an approach requires storing gradients for multiple sampling timesteps, leading to considerable time and memory costs. To address this, we introduce an efficient reward strategy that deliberately disturbs the input images by adding noise, and then uses the single-step denoised images for reward fine-tuning. This avoids the extensive costs associated with image sampling, allowing for more efficient reward fine-tuning. Extensive experiments show that ControlNet++ significantly improves controllability under various conditional controls. For example, it achieves improvements over ControlNet by 7.9% mIoU, 13.4% SSIM, and 7.6% RMSE, respectively, for segmentation mask, line-art edge, and depth conditions.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {BAMM: Bidirectional Autoregressive Motion Model},

author = {Ekkasit Pinyoanuntapong and Muhammad Usama Saleem and Pu Wang and Minwoo Lee and Srijan Das and Chen Chen},

url = {https://arxiv.org/pdf/2403.19435.pdf

https://exitudio.github.io/BAMM-page/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Generating human motion from text has been dominated by denoising motion models either through diffusion or generative masking process. However, these models face great limitations in usability by requiring prior knowledge of the motion length. Conversely, autoregressive motion models address this limitation by adaptively predicting motion endpoints, at the cost of degraded generation quality and editing capabilities. To address these challenges, we propose Bidirectional Autoregressive Motion Model (BAMM), a novel text-to-motion generation framework. BAMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into discrete tokens in latent space, and (2) a masked self-attention transformer that autoregressively predicts randomly masked tokens via a hybrid attention masking strategy. By unifying generative masked modeling and autoregressive modeling, BAMM captures rich and bidirectional dependencies among motion tokens, while learning the probabilistic mapping from textual inputs to motion outputs with dynamically-adjusted motion sequence length. This feature enables BAMM to simultaneously achieving high-quality motion generation with enhanced usability and built-in motion editability. Extensive experiments on HumanML3D and KIT-ML datasets demonstrate that BAMM surpasses current state-of-the-art methods in both qualitative and quantitative measures.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024,

title = {3DEgo: 3D Editing on the Go!},

author = {Umar Khalid and Hasan Iqbal and Azib Farooq and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2407.10102

https://3dego.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {We introduce 3DEgo to address a novel problem of directly synthesizing photorealistic 3D scenes from monocular videos guided by textual prompts. Conventional methods construct a text-conditioned 3D scene through a three-stage process, involving pose estimation using Structure-from-Motion (SfM) libraries like COLMAP, initializing the 3D model with unedited images, and iteratively updating the dataset with edited images to achieve a 3D scene with text fidelity. Our framework streamlines the conventional multi-stage 3D editing process into a single-stage workflow by overcoming the reliance on COLMAP and eliminating the cost of model initialization. We apply a diffusion model to edit video frames prior to 3D scene creation by incorporating our designed noise blender module for enhancing multi-view editing consistency, a step that does not require additional training or fine-tuning of T2I diffusion models. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the multi-view consistent edited frames, capitalizing on the inherent temporal continuity and explicit point cloud data. 3DEgo demonstrates remarkable editing precision, speed, and adaptability across a variety of video sources, as validated by extensive evaluations on six datasets, including our own prepared GS25 dataset.



},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024b,

title = {LatentEditor: Text Driven Local Editing of 3D Scenes},

author = {Umar Khalid and Hasan Iqbal and Muhammad Tayyab and Md Nazmul Karim and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2312.09313.pdf

https://latenteditor.github.io/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {While neural fields have made significant strides in view synthesis and scene reconstruction, editing them poses a formidable challenge due to their implicit encoding of geometry and texture information from multi-view inputs. In this paper, we introduce LatenEditor, an innovative framework designed to empower users with the ability to perform precise and locally controlled editing of neural fields using text prompts. Leveraging denoising diffusion models, we successfully embed real-world scenes into the latent space, resulting in a faster and more adaptable NeRF backbone for editing compared to traditional methods. To enhance editing precision, we introduce a delta score to calculate the 2D mask in the latent space that serves as a guide for local modifications while preserving irrelevant regions. Our novel pixel-level scoring approach harnesses the power of InstructPix2Pix (IP2P) to discern the disparity between IP2P conditional and unconditional noise predictions in the latent space. The edited latents conditioned on the 2D masks are then iteratively updated in the training set to achieve 3D local editing. Our approach achieves faster editing speeds and superior output quality compared to existing 3D editing models, bridging the gap between textual instructions and high-quality 3D scene editing in latent space.



},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Karim2024,

title = {Free-Editor: Zero-shot Text-driven 3D Scene Editing},

author = {Md Nazmul Karim and Hasan Iqbal and Umar Khalid and Chen Chen and Jing Hua

},

url = {https://arxiv.org/pdf/2312.13663.pdf

https://free-editor.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Text-to-Image (T2I) diffusion models have gained popularity recently due to their multipurpose and easy-to-use nature, e.g. image and video generation as well as editing. However, training a diffusion model specifically for 3D scene editing is not straightforward due to the lack of large-scale datasets. To date, editing 3D scenes requires either re-training the model to adapt to various 3D edited scenes or design-specific methods for each special editing type. Furthermore, state-of-the-art (SOTA) methods require multiple synchronized edited images from the same scene to facilitate the scene editing. Due to the current limitations of T2I models, it is very challenging to apply consis- tent editing effects to multiple images, i.e. multi-view inconsistency in editing. This in turn compromises the desired 3D scene editing performance if these images are used. In our work, we propose a novel training-free 3D scene editing technique, FREE-EDITOR, which allows users to edit 3D scenes without further re-training the model during test time. Our proposed method successfully avoids the multi- view style inconsistency issue in SOTA methods with the help of a “single-view editing” scheme. Specifically, we show that editing a particular 3D scene can be performed by only modifying a single view. To this end, we introduce an Edit Transformer that enforces intra-view consistency and inter-view style transfer by utilizing self- and cross- attention, respectively. Since it is no longer required to re-train the model and edit every view in a scene, the editing time, as well as memory resources, are reduced significantly, e.g., the runtime being ∼ 20× faster than SOTA. We have conducted extensive experiments on a wide range of benchmark datasets and achieve diverse editing capabilities with our proposed technique.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fang2024,

title = {Rethinking Weakly-supervised Video Temporal Grounding From a Game Perspective},

author = {Xiang Fang and Zeyu Xiong and Wanlong Fang and Xiaoye Qu and Chen Chen and Jianfeng Dong and Keke Tang and Pan Zhou and Yu Cheng and Daizong Liu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ECCV2024_Grounding_camera.pdf

https://eccv2024.ecva.net/virtual/2024/poster/1833},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses the challenging task of weakly-supervised video temporal grounding. Existing approaches are generally based on the moment candidate selection pipeline that utilizes contrastive learning and reconstruction paradigm for scoring the pre-defined moments. Although they have achieved significant progress, we argue that their current frameworks have overlooked two indispensable issues: (1) Coarse-grained cross-modal learning: previous methods solely capture the global video-level alignment with the query, failing to model the detailed consistency between video frames and query words for accurately grounding the moment boundaries. (2) Complex moment candidates: the performance of these methods severely relies on the quality of moment candidates, which are also time-consuming and complicated for selection. To this end, in this paper, we make the first attempt to tackle this task from a novel game perspective, which effectively learns the uncertain relationship between each frame-word pair with diverse granularity and flexible combination for fine-grained cross-modal interaction. Specifically, we creatively model each video frame and query word as game players with multivariate cooperative game theory to learn their contribution to the cross-modal similarity score. By quantifying the trend of frame-word cooperation within a coalition via the game-theoretic interaction, we are able to value all uncertain but possible correspondence between frames and words. At last, instead of using moment proposals, we utilize the learned query-guided frame-wise scores for fine-grained moment boundary grounding. Experiments show that our method achieves superior performance on both Charades-STA and ActivityNet Caption datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf

https://arxiv.org/abs/2408.02840

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf

https://parthpk.github.io/cityguessr-webpage/

},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38]. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Augmented Neural Fine-tuning for Efficient Backdoor Purification},

author = {Nazmul Karim and Abdullah Al Arafat and Umar Khalid and Zhishan Guo and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2024_ECCV_NFT.pdf

https://arxiv.org/pdf/2407.10052

https://github.com/nazmul-karim170/NFT-Augmented-Backdoor-Purification},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent studies have revealed the vulnerability of deep neural networks (DNNs) to various backdoor attacks, where the behavior of DNNs can be compromised by utilizing certain types of triggers or poisoning mechanisms. State-of-the-art (SOTA) defenses employ too sophisticated mechanisms that require either a computationally expensive adversarial search module for reverse-engineering the trigger distribution or an over-sensitive hyper-parameter selection module. Moreover, they offer sub-par performance in challenging scenarios, e.g., limited validation data and strong attacks. In this paper, we propose—Neural mask Fine-Tuning (NFT)—with an aim to optimally re-organize the neuron activities in a way that the effect of the backdoor is removed. Utilizing a simple data augmentation like MixUp, NFT relaxes the trigger synthesis process and eliminates the requirement of the adversarial search module. Our study further reveals that direct weight fine-tuning under limited validation data results in poor post-purification clean test accuracy, primarily due to overfitting issue. To overcome this, we propose to fine-tune neural masks instead of model weights. In addition, a mask regularizer

has been devised to further mitigate the model drift during the purification process. The distinct characteristics of NFT render it highly efficient in both runtime and sample usage, as it can remove the backdoor even when a single sample is available from each class. We validate the effectiveness of NFT through extensive experiments covering the tasks of image classification, object detection, video action recognition, 3D point cloud, and natural language processing. We evaluate our method against 14 different attacks (LIRA, WaNet, etc.) on 11 benchmark data sets (ImageNet, UCF101, Pascal VOC, ModelNet, OpenSubtitles2012, etc.). },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2024b,

title = {A Dual-Augmentor Framework for Domain Generalization in 3D Human Pose Estimation Conference},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.computer.org/csdl/proceedings-article/cvpr/2024/530000c240/20hM2VBrKik},

doi = {10.1109/CVPR52733.2024.00218},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {3D human pose data collected in controlled laboratory settings present challenges for pose estimators that generalize across diverse scenarios. To address this, domain generalization is employed. Current methodologies in do-main generalization for 3D human pose estimation typically utilize adversarial training to generate synthetic poses for training. Nonetheless, these approaches exhibit several limitations. First, the lack of prior information about the target domain complicates the application of suitable augmentation through a single pose augmentor, affecting generalization on target domains. Moreover, adversarial training's discriminator tends to enforce similarity between source and synthesized poses, impeding the exploration of out-of-source distributions. Furthermore, the pose estimator's op-timization is not exposed to domain shifts, limiting its over-all generalization ability. To address these limitations, we propose a novel frame-work featuring two pose augmentors: the weak and the strong augmentors. Our framework employs differential strategies for generation and discrimination processes, facilitating the preservation of knowledge related to source poses and the exploration of out-of-source distributions without prior information about target poses. Besides, we leverage meta-optimization to simulate domain shifts in the optimization process of the pose estimator, thereby improving its generalization ability. Our proposed approach significantly outperforms existing methods, as demonstrated through comprehensive experiments on various benchmark datasets. Our code will be released at https://github.com/davidpengucf/DAF-DG.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2024,

title = {A Dual-Augmentor Framework for Domain Generalization in 3D Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.computer.org/csdl/proceedings-article/cvpr/2024/530000c240/20hM2VBrKik

https://github.com/davidpengucf/DAF-DG

},

doi = {10.1109/CVPR52733.2024.00218},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {3D human pose data collected in controlled laboratory settings present challenges for pose estimators that generalize across diverse scenarios. To address this, domain generalization is employed. Current methodologies in domain generalization for 3D human pose estimation typically utilize adversarial training to generate synthetic poses for training. Nonetheless, these approaches exhibit several limitations. First, the lack of prior information about the target domain complicates the application of suitable augmentation through a single pose augmentor, affecting generalization on target domains. Moreover, adversarial training’s discriminator tends to enforce similarity between source and synthesized poses, impeding the exploration of out-ofsource distributions. Furthermore, the pose estimator’s optimization is not exposed to domain shifts, limiting its overall generalization ability. 

 

To address these limitations, we propose a novel framework featuring two pose augmentors: the weak and the strong augmentors. Our framework employs differential strategies for generation and discrimination processes, facilitating the preservation of knowledge related to source poses and the exploration of out-of-source distributions without prior information about target poses. Besides, we leverage meta-optimization to simulate domain shifts in the optimization process of the pose estimator, thereby improving its generalization ability. Our proposed approach significantly outperforms existing methods, as demonstrated through comprehensive experiments on various benchmark datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2024,

title = {Multi-View Attentive Contextualization for Multi-View 3D Object Detection},

author = {Xianpeng Liu and Ce Zheng and Ming Qian and Nan Xue and Chen Chen and Zhebin Zhang and Chen Li and Tianfu Wu},

year  = {2024},

date = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yuan2024,

title = {Towards Surveillance Video-and-Language Understanding: New Dataset, Baselines, and Challenges},

author = {Tongtong Yuan and Xuange Zhang and Kun Liu and Bo Liu and Chen Chen and Jian Jin and Zhenzhen Jiao},

url = {https://cvpr.thecvf.com/virtual/2024/poster/30807

https://xuange923.github.io/Surveillance-Video-Understanding

https://github.com/Xuange923/Surveillance-Video-Understanding},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Surveillance videos are an essential component of daily life with various critical applications, particularly in public security. However, current surveillance video tasks mainly focus on classifying and localizing anomalous events. Existing methods are limited to detecting and classifying the predefined events with unsatisfactory semantic understanding, although they have obtained considerable performance. To address this issue, we propose a new research direction of surveillance video-and-language understanding, and construct the first multimodal surveillance video dataset. We manually annotate the real-world surveillance dataset UCF-Crime with fine-grained event content and timing. Our newly annotated dataset, UCA UCF-Crime Annotation), contains 23,542 sentences, with an average length of 20 words, and its annotated videos are as long as 110.7 hours. Furthermore, we benchmark SOTA models for four multimodal tasks on this newly created dataset, which serve as new baselines for surveillance video-and-language understanding. Through our experiments, we find that mainstream models used in previously publicly available datasets perform poorly on surveillance video, which demonstrates the new challenges in surveillance video-and-language understanding. To validate the effectiveness of our UCA, we conducted experiments on multimodal anomaly detection. The results demonstrate that our multimodal surveillance learning can improve the performance of conventional anomaly detection tasks. All the experiments highlight the necessity of constructing this dataset to advance surveillance AI.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pinyoanuntapong2024,

title = {MMM: Generative Masked Motion Model},

author = {Ekkasit Pinyoanuntapong and Pu Wang and Minwoo Lee and Chen Chen},

url = {https://arxiv.org/pdf/2312.03596.pdf

https://exitudio.github.io/MMM-page/

https://github.com/exitudio/MMM/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in text-to-motion generation using diffusion and autoregressive models have shown promising results. However, these models often suffer from a trade-off between real-time performance, high fidelity, and motion editability. To address this gap, we introduce MMM, a novel yet simple motion generation paradigm based on Masked Motion Model. MMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into a sequence of discrete tokens in latent space, and (2) a conditional masked motion transformer that learns to predict randomly masked motion tokens, conditioned on the pre-computed text tokens. By attending to motion and text tokens in all directions, MMM explicitly captures inherent dependency among motion tokens and semantic mapping between motion and text tokens. During inference, this allows parallel and iterative decoding of multiple motion tokens that are highly consistent with fine-grained text descriptions, therefore simultaneously achieving high-fidelity and high-speed motion generation. In addition, MMM has innate motion editability. By simply placing mask tokens in the place that needs editing, MMM automatically fills the gaps while guaranteeing smooth transitions between editing and non-editing parts. Extensive experiments on the HumanML3D and KIT-ML datasets demonstrate that MMM surpasses current leading methods in generating high-quality motion (evidenced by superior FID scores of 0.08 and 0.429), while offering advanced editing features such as body-part modification, motion in-betweening, and the synthesis of long motion sequences. In addition, MMM is two orders of magnitude faster on a single mid-range GPU than editable motion diffusion models.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024,

title = {Skeleton-in-Context: Unified Skeleton Sequence Modeling with In-Context Learning},

author = {Xinshun Wang and Zhongbin Fang and Xia Li, Xiangtai Li and Chen Chen and Mengyuan Liu},

url = {https://arxiv.org/pdf/2312.03703.pdf

https://bradleywang0416.github.io/skeletonincontext/

https://github.com/fanglaosi/Skeleton-in-Context},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In-context learning provides a new perspective for multi-task modeling for vision and NLP. Under this setting, the model can perceive tasks from prompts and accomplish them without any extra task-specific head predictions or model finetuning. However, Skeleton sequence modeling via in-context learning remains unexplored. Directly applying existing in-context models from other areas onto skeleton sequences fails due to the inter-frame and cross-task pose similarity that makes it outstandingly hard to perceive the task correctly from a subtle context.



To address this challenge, we propose Skeleton-in-Context (SiC), an effective framework for in-context skeleton sequence modeling. Our SiC is able to handle multiple skeleton-based tasks simultaneously after a single training process and accomplish each task from context according to the given prompt. It can further generalize to new, unseen tasks according to customized prompts. To facilitate context perception, we additionally propose a task-unified prompt, which adaptively learns tasks of different natures, such as partial joint-level generation, sequence-level prediction, or 2D-to-3D motion prediction. We conduct extensive experiments to evaluate the effectiveness of our SiC on multiple tasks, including motion prediction, pose estimation, joint completion, and future pose estimation. We also evaluate its generalization capability on unseen tasks such as motion-in-between. These experiments show that our model achieves state-of-the-art multi-task performance and even outperforms single-task methods on certain tasks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024,

title = {OST: Refining Text Knowledge with Optimal Spatio-Temporal Descriptor for General Video Recognition},

author = {Tongjia Chen and Hongshan Yu and Zhengeng Yang and Zechuan Li and Wei Sun and Chen Chen},

url = {https://arxiv.org/pdf/2312.00096.pdf

https://tomchen-ctj.github.io/OST/

https://github.com/tomchen-ctj/OST},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Due to the resource-intensive nature of training vision-language models on expansive video data, a majority of studies have centered on adapting pre-trained image-language models to the video domain. Dominant pipelines propose to tackle the visual discrepancies with additional temporal learners while overlooking the substantial discrepancy for web-scaled descriptive narratives and concise action category names, leading to less distinct semantic space and potential performance limitations. In this work, we prioritize the refinement of text knowledge to facilitate generalizable video recognition. To address the limitations of the less distinct semantic space of category names, we prompt a large language model (LLM) to augment action class names into Spatio-Temporal Descriptors thus bridging the textual discrepancy and serving as a knowledge base for general recognition. Moreover, to assign the best descriptors with different video instances, we propose Optimal Descriptor Solver, forming the video recognition problem as solving the optimal matching flow across frame-level representations and descriptors. Comprehensive evaluations in zero-shot, few-shot, and fully supervised video recognition highlight the effectiveness of our approach. Our best model achieves a state-of-the-art zero-shot accuracy of 75.1% on Kinetics-600.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2024,

title = {VidLA: Video-Language Alignment at Scale},

author = {Mamshad Nayeem Rizve and Fan Fei and Jayakrishnan Unnikrishnan and Son Tran and Benjamin Z. Yao and Belinda Zeng and Mubarak Shah and Trishul Chilimbi},

url = {https://arxiv.org/abs/2403.14870},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we propose VidLA, an approach for video-language alignment at scale. There are two major limitations of previous video-language alignment approaches. First, they do not capture both short-range and long-range temporal dependencies and typically employ complex hierarchical deep network architectures that are hard to integrate with existing pretrained image-text foundation models. To effectively address this limitation, we instead keep the network architecture simple and use a set of data tokens that operate at different temporal resolutions in a hierarchical manner, accounting for the temporally hierarchical nature of videos. By employing a simple two-tower architecture, we are able to initialize our video-language model with pretrained image-text foundation models, thereby boosting the final performance. Second, existing video-language alignment works struggle due to the lack of semantically aligned large-scale training data. To overcome it, we leverage recent LLMs to curate the largest video-language dataset to date with better visual grounding. Furthermore, unlike existing video-text datasets which only contain short clips, our dataset is enriched with video clips of varying durations to aid our temporally hierarchical data tokens in extracting better representations at varying temporal scales. Overall, empirical results show that our proposed approach surpasses state-of-the-art methods on multiple retrieval benchmarks, especially on longer videos, and performs competitively on classification benchmarks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ristea2024,

title = {Self-Distilled Masked Auto-Encoders are Efficient Video Anomaly Detectors},

author = {Nicolae Catalin Ristea and Florinel Alin Croitoru and Radu Tudor Ionescu and Marius Popescu and Fahad Khan and Mubarak Shah},

url = {https://arxiv.org/abs/2306.12041

https://github.com/ristea/aed-mae},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {We propose an efficient abnormal event detection model based on a lightweight masked auto-encoder (AE) applied at the video frame level. The novelty of the proposed model is threefold. First, we introduce an approach to weight tokens based on motion gradients, thus shifting the focus from the static background scene to the foreground objects. Second, we integrate a teacher decoder and a student decoder into our architecture, leveraging the discrepancy between the outputs given by the two decoders to improve anomaly detection. Third, we generate synthetic abnormal events to augment the training videos, and task the masked AE model to jointly reconstruct the original frames (without anomalies) and the corresponding pixel-level anomaly maps. Our design leads to an efficient and effective model, as demonstrated by the extensive experiments carried out on four benchmarks: Avenue, ShanghaiTech, UBnormal and UCSD Ped2. The empirical results show that our model achieves an excellent trade-off between speed and accuracy, obtaining competitive AUC scores, while processing 1655 FPS. Hence, our model is between 8 and 70 times faster than competing methods. We also conduct an ablation study to justify our design. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2024,

title = {Composed Video Retrieval via Enriched Context and Discriminative Embeddings},

author = {Omkar Chakradhar Thawakar and Muzammal Naseer and Rao Muhammad Anwer and Salman Khan and Michael Felsberg and Mubarak Shah and Fahad Khan },

url = {https://arxiv.org/abs/2403.16997

https://github.com/OmkarThawakar/composed-video-retrieval},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Composed video retrieval (CoVR) is a challenging problem in computer vision which has recently highlighted the integration of modification text with visual queries for more sophisticated video search in large databases. Existing works predominantly rely on visual queries combined with modification text to distinguish relevant videos. However, such a strategy struggles to fully preserve the rich query-specific context in retrieved target videos and only represents the target video using visual embedding. We introduce a novel CoVR framework that leverages detailed language descriptions to explicitly encode query-specific contextual information and learns discriminative embeddings of vision only, text only and vision-text for better alignment to accurately retrieve matched target videos. Our proposed framework can be flexibly employed for both composed video (CoVR) and image (CoIR) retrieval tasks. Experiments on three datasets show that our approach obtains state-of-the-art performance for both CovR and zero-shot CoIR tasks, achieving gains as high as around 7% in terms of recall@K=1 score.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dutta2024,

title = {Multiview Aerial Visual RECognition (MAVREC) Dataset: Can Multi-view Improve Aerial Visual Perception?},

author = {Aritra Dutta and Srijan Das and Jacob Nielsen and Rajatsubhra Chakraborty and Mubarak Shah},

url = {https://arxiv.org/abs/2312.04548

https://mavrec.github.io/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite the commercial abundance of UAVs, aerial data acquisition remains challenging, and the existing Asia and North America-centric open-source UAV datasets are small-scale or low-resolution and lack diversity in scene contextuality. Additionally, the color content of the scenes, solar-zenith angle, and population density of different geographies influence the data diversity. These two factors conjointly render suboptimal aerial-visual perception of the deep neural network (DNN) models trained primarily on the ground-view data, including the open-world foundational models.



To pave the way for a transformative era of aerial detection, we present Multiview Aerial Visual RECognition or MAVREC, a video dataset where we record synchronized scenes from different perspectives -- ground camera and drone-mounted camera. MAVREC consists of around 2.5 hours of industry-standard 2.7K resolution video sequences, more than 0.5 million frames, and 1.1 million annotated bounding boxes. This makes MAVREC the largest ground and aerial-view dataset, and the fourth largest among all drone-based datasets across all modalities and tasks. Through our extensive benchmarking on MAVREC, we recognize that augmenting object detectors with ground-view images from the corresponding geographical location is a superior pre-training strategy for aerial detection. Building on this strategy, we benchmark MAVREC with a curriculum-based semi-supervised object detection approach that leverages labeled (ground and aerial) and unlabeled (only aerial) images to enhance the aerial detection.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2024,

title = {Activity-Biometrics: Person Identification from Daily Activities},

author = {Shehreen Azad and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2403.17360

https://github.com/sacrcv/Activity-Biometrics/

},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {In this work, we study a novel problem which focuses on person identification while performing daily activities. Learning biometric features from RGB videos is challenging due to spatio-temporal complexity and presence of appearance biases such as clothing color and background. We propose ABNet, a novel framework which leverages disentanglement of biometric and non-biometric features to perform effective person identification from daily activities. ABNet relies on a bias-less teacher to learn biometric features from RGB videos and explicitly disentangle non-biometric features with the help of biometric distortion. In addition, ABNet also exploits activity prior for biometrics which is enabled by joint biometric and activity learning. We perform comprehensive evaluation of the proposed approach across five different datasets which are derived from existing activity recognition benchmarks. Furthermore, we extensively compare ABNet with existing works in person identification and demonstrate its effectiveness for activity-based biometrics across all five datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Frakes2024,

title = {Efficient and consistent zero-shot video generation with diffusion models},

author = {Ethan Frakes and Umar Khalid and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Report-Ethan-Frakes.pdf

https://spie.org/defense-commercial-sensing/presentation/Efficient-and-consistent-zero-shot-video-generation-with-diffusion-models/13034-8#_=_},

year  = {2024},

date = {2024-04-22},

urldate = {2024-04-22},

publisher = {SPIE Defense+ Commercial Sensing},

abstract = {Recent diffusion-based generative models employ methods such as one-shot fine-tuning an image diffusion model for video generation. However, this leads to long video generation times and suboptimal efficiency. To resolve this long generation time, zero-shot text-to-video models eliminate the fine-tuning method entirely and can generate novel videos from a text prompt alone. While the zero-shot generation method greatly reduces generation time, many models rely on inefficient cross-frame attention processors, hindering the diffusion model’s utilization for real-time video generation. We address this issue by introducing more efficient attention processors to a video diffusion model. Specifically, we use attention processors (i.e. xFormers, FlashAttention, and HyperAttention) that are highly optimized for efficiency and hardware parallelization. We then apply these processors to a video generator and test with both older diffusion models such as Stable Diffusion 1.5 and newer, high-quality models such as Stable Diffusion XL. Our results show that using efficient attention processors alone can reduce generation time by around 25%, while not resulting in any change in video quality. Combined with the use of higher quality models, this use of efficient attention processors in zero-shot generation presents a substantial efficiency and quality increase, greatly expanding the video diffusion model’s application to real-time video generation.},

keywords = {REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024c,

title = {CodaMal: Contrastive Domain Adaptation for Malaria Detection in Low-Cost Microscopes },

author = {Ishan Rajendrakumar Dave and Tristan de Blegiers and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2402.10478

https://daveishan.github.io/codamal-webpage/},

year  = {2024},

date = {2024-02-16},

publisher = {31st IEEE International Conference on Image Processing (ICIP) (Oral)},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {No More Shortcuts: Realizing the Potential of Temporal Self-Supervision},

author = {Ishan Rajendrakumar Dave and Simon Jenni and Mubarak Shah},

url = {https://arxiv.org/pdf/2312.13008

https://daveishan.github.io/nms-webpage/

https://youtu.be/5MBnxmMBQh0},

year  = {2023},

date = {2023-12-20},

publisher = {AAAI Conference on Artificial Intelligence (AAAI), Main Technical Track},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cepeda2023,

title = {GeoCLIP: Clip-Inspired Alignment between Locations and Images for Effective Worldwide Geo-localization},

author = {Vicente Vivanco Cepeda and Gaurav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_paper.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_supplementary.pdf

https://vicentevivan.github.io/GeoCLIP/},

year  = {2023},

date = {2023-12-11},

publisher = {Thirty-seventh Conference on Neural Information Processing Systems},

abstract = {Worldwide Geo-localization aims to pinpoint the precise location of images taken anywhere on Earth. This task has considerable challenges due to immense variation in geographic landscapes. The image-to-image retrieval-based approaches fail to solve this problem on a global scale as it is not feasible to construct a large gallery of images covering the entire world. Instead, existing approaches divide the globe into discrete geographic cells, transforming the problem into a classification task. However, their performance is limited by the predefined classes and often results in inaccurate localizations when an image’s location significantly deviates from its class center. To overcome these limitations, we propose GeoCLIP, a novel CLIP-inspired Image-to-GPS retrieval approach that enforces alignment between the image and its corresponding GPS locations. GeoCLIP’s location encoder models the Earth as a continuous function by employing positional encoding through random Fourier features and constructing a hierarchical representation that captures information at varying resolutions to yield a semantically rich highdimensional feature suitable to use even beyond geo-localization. To the best of our knowledge, this is the first work employing GPS encoding for geo-localization. We demonstrate the efficacy of our method via extensive experiments and ablations on benchmark datasets. We achieve competitive performance with just 20% of training data, highlighting its effectiveness even in limited-data settings. Furthermore, we qualitatively demonstrate geo-localization using a text query by leveraging CLIP backbone of our image encoder.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2023,

title = {On Occlusions in Video Action Detection: Benchmark Datasets And Training Recipes},

author = {Rajat Modi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/nips_23.pdf},

year  = {2023},

date = {2023-12-10},

publisher = {NeurIPS 2023.},

abstract = {This paper explores the impact of occlusions in video action detection. We facilitate this study by introducing five new benchmark datasets namely O-UCF and OJHMDB consisting of synthetically controlled static/dynamic occlusions, OVISUCF and OVIS-JHMDB consisting of occlusions with realistic motions and Real-OUCF for occlusions in realistic-world scenarios. We formally confirm an intuitive expectation: existing models suffer a lot as occlusion severity is increased and exhibit different behaviours when occluders are static vs when they are  moving. We discover several curious phenomenon emerging in neural nets: 1) transformers  can naturally outperform CNN models which might have even used occlusion as a form of data augmentation during training 2) incorporating symbolic-components like capsules to such backbones allows them to bind to occluders never even seen during training and 3) Islands of agreement (similar to the ones hypothesized in Hinton et Al’s GLOM) can emerge in realistic images/videos without instance-level supervision, distillation or contrastive-based objectives2(eg. video-textual training). Such emergent properties allow us to derive simple yet effective training recipes which lead to robust occlusion models inductively satisfying the first two stages of the binding mechanism (grouping/segregation). Models leveraging these recipes outperform existing video action-detectors under occlusion by 32.3% on O-UCF, 32.7% on O-JHMDB & 2.6% on Real-OUCF in terms of the vMAP metric.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Frequency Domain Adversarial Training for Robust Volumetric Medical Segmentation},

author = {Asif Hanif and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Frequency-Domain-Adversarial-Training-for-Robust-Volumetric-Medical-Segmentation.pdf

https://github.com/asif-hanif/vafa},

doi = {https://doi.org/10.48550/arXiv.2307.07269},

year  = {2023},

date = {2023-10-08},

publisher = {The 26th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2023},

keywords = {MICCAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2023,

title = {AlignDet: Aligning Pre-training and Fine-tuning in Object Detection},

author = {Ming Li and Jie Wu and Xionghui Wang and Chen Chen and Jie Qin and Xuefeng Xiao and Rui Wang and Min Zheng and Xin Pan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.11077.pdf

https://arxiv.org/abs/2307.11077

https://github.com/liming-ai/AlignDet

https://openreview.net/forum?id=8PA2nX9v_r2

https://liming-ai.github.io/AlignDet/},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2023,

title = {Source-free Domain Adaptive Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.03202.pdf

https://arxiv.org/abs/2308.03202

https://github.com/davidpengucf/SFDAHPE},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Human Pose Estimation (HPE) is widely used in various fields, including motion analysis, healthcare, and virtual reality. However, the great expenses of labeled real-world datasets present a significant challenge for HPE. To  overcome this, one approach is to train HPE models on synthetic datasets and then perform domain adaptation (DA) on realworld data. Unfortunately, existing DA methods for HPE neglect data privacy and security by using both source and target data in the adaptation process. To this end, we propose a new task, named sourcefree domain adaptive HPE, which aims to address the challenges of cross-domain learning of HPE without access to source data during the adaptation process. We further propose a novel framework that consists of three models: source model, intermediate model, and target model, which explores the task from both sourceprotect and target-relevant perspectives. The sourceprotect module preserves source information more effectively while resisting noise, and the target-relevant module reduces the sparsity of spatial representations by building a novel spatial probability space, and pose-specific contrastive learning and information maximization are proposed on the basis of this space. Comprehensive experiments on several domain adaptive HPE benchmarks show that the proposed method outperforms existing approaches by a considerable margin. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {FedPerfix: Towards Partial Model Personalization of Vision Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.09160.pdf

https://arxiv.org/abs/2308.09160

https://github.com/imguangyu/FedPerfix},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We propose and analyze a general framework of federated learning with  partial model personalization. Compared with full model personalization, partial model personalization relies on domain knowledge to select a small portion of the model to personalize, thus imposing a much smaller on-device memory footprint. We propose two federated optimization algorithms for training partially personalized models, where the shared and personal parameters are updated either simultaneously or alternately on each device, but only the shared parameters are communicated and aggregated at the server. We give convergence analyses of both algorithms for minimizing smooth nonconvex functions, providing theoretical support of them for training deep learning models. Our experiments on real-world image and text datasets demonstrate that (a) partial model personalization can obtain most of the benefit of full model personalization with a small fraction of personalized parameters, and, (b) the alternating update algorithm often outperforms the simultaneous update algorithm.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PGFed: Personalize Each Client's Global Objective for Federated Learning},

author = {Jun Luo and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.01448.pdf

https://github.com/ljaiverson/pgfed},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The mediocre performance of conventional federated learning (FL) over heterogeneous data has been facilitating personalized FL solutions, where, unlike conventional FL which trains a single global consensus model, different models are allowed for different clients. However, in most existing personalized FL algorithms, the collaborative knowledge across the federation was only implicitly passed to the clients in ways such as model aggregation or regularization. We observed that this implicit knowledge transfer fails to maximize the potential value of each client's empirical risk toward other clients. Based on our observation, in this work, we propose Personalized Global Federated Learning (PGFed), a novel personalized FL framework that enables each client to personalize its own global objective by explicitly and adaptively aggregating the empirical risks of itself and other clients. To avoid massive (O(N2)) communication overhead and potential privacy leakage, each client's risk is estimated through a first-order approximation for other clients' adaptive risk aggregation. On top of PGFed, we develop a momentum upgrade, dubbed PGFedMo, to more efficiently utilize clients' empirical risks. Our extensive experiments under different federated settings with benchmark datasets show consistent improvements of PGFed over the compared state-of-the-art alternatives.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {A Large-scale Study of Spatiotemporal Representation Learning with a New Benchmark on Action Recognition},

author = {Andong Dang and Taojiannan Yang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.13505.pdf

https://arxiv.org/abs/2303.13505

https://github.com/AndongDeng/BEAR},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The goal of building a benchmark (suite of datasets) is to provide a unified protocol for fair evaluation and thus facilitate the evolution of a specific area. Nonetheless, we point out that existing protocols of action recognition could yield partial evaluations due to several limitations. To comprehensively probe the effectiveness of spatiotemporal representation learning, we introduce BEAR, a new BEnchmark on video Action Recognition. BEAR is a collection of 18 video datasets grouped into 5 categories (anomaly, gesture, daily, sports, and instructional), which covers a diverse set of real-world applications. With BEAR, we thoroughly evaluate 6 common spatiotemporal models pre-trained by both supervised and self-supervised learning. We also report transfer performance via standard finetuning, few-shot finetuning, and unsupervised domain adaptation. Our observation suggests that current state-of-the-art cannot solidly guarantee high performance on datasets close to real-world applications, and we hope BEAR can serve as a fair and challenging evaluation benchmark to gain insights on building next-generation spatiotemporal learners. Our dataset, code, and models are released at: https://github.com/AndongDeng/BEAR},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Towards Geospatial Foundation Models via Continual Pretraining},

author = {Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2302.04476.pdf

https://arxiv.org/abs/2302.04476

https://github.com/mmendiet/GFM},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Geospatial technologies are becoming increasingly essential in our world for a wide range of applications, including agriculture, urban planning, and disaster response. To help improve the applicability and performance of deep learning models on these geospatial tasks, various works have begun investigating foundation models for this domain. Researchers have explored two prominent approaches for introducing such models in geospatial applications, but both have drawbacks in terms of limited performance benefit or prohibitive training cost. Therefore, in this work, we propose a novel paradigm for building highly effective geospatial foundation models with minimal resource cost and carbon impact. We first construct a compact yet diverse dataset from multiple sources to promote feature diversity, which we term GeoPile. Then, we investigate the potential of continual pretraining from large-scale ImageNet-22k models and propose a multi-objective continual pretraining paradigm, which leverages the strong representations of ImageNet while simultaneously providing the freedom to learn valuable in-domain features. Our approach outperforms previous state-of-the-art geospatial pretraining methods in an extensive evaluation on seven downstream datasets covering various tasks such as change detection, classification, multi-label classification, semantic segmentation, and super-resolution.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Multi-view Self-supervised Disentanglement for General Image Denoising},

author = {Hao Chen and Chenyuan Qu and Yu Zhang and Chen Chen and Jianbo Jiao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Final_Version.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Supplymentary_Final_Version.pdf

https://chqwer2.github.io/MeD/

https://github.com/chqwer2/Multi-view-Self-supervised-Disentanglement-Denoising},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {With its significant performance improvements, the deep

learning paradigm has become a standard tool for modern

image denoisers. While promising performance has been

shown on seen noise distributions, existing approaches often suffer from generalisation to unseen noise types or general and real noise. It is understandable as the model is

designed to learn paired mapping (e.g. from a noisy image

to its clean version). In this paper, we instead propose to

learn to disentangle the noisy image, under the intuitive

assumption that different corrupted versions of the same

clean image share a common latent space. A self-supervised

learning framework is proposed to achieve the goal, without looking at the latent clean image. By taking two different corrupted versions of the same image as input, the proposed Multi-view Self-supervised Disentanglement (MeD) approach learns to disentangle the latent clean features from the corruptions and recover the clean image consequently. Extensive experimental analysis on both synthetic and real noise shows the superiority of the proposed method over prior self-supervised approaches, especially on unseen novel noise types. On real noise, the proposed method even outperforms its supervised counterparts by over 3 dB. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {RenderIH: A large-scale synthetic dataset for 3D interacting hand pose estimation},

author = {Lijun Li, Linrui Tian and Xindi Zhang and Qi Wang and Bang Zhang and Liefeng Bo and Mengyuan Liu and Chen Chen},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023b,

title = {When Do Curricula Work in Federated Learning? },

author = {Saeed Vahidian and Sreevatsank Kadaveru and Woonjoon Baek and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.12712.pdf

https://arxiv.org/abs/2212.12712},

doi = {https://doi.org/10.48550/arXiv.2212.12712},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {An oft-cited open problem of federated learning is the existence of data heterogeneity at the clients. One pathway to understanding the drastic accuracy drop in federated learning is by scrutinizing the behavior of the clients' deep models on data with different levels of "difficulty", which has been left unaddressed. In this paper, we investigate a different and rarely studied dimension of FL: ordered learning. Specifically, we aim to investigate how ordered learning principles can contribute to alleviating the heterogeneity effects in FL. We present theoretical analysis and conduct extensive empirical studies on the efficacy of orderings spanning three kinds of learning: curriculum, anti-curriculum, and random curriculum. We find that curriculum learning largely alleviates non-IIDness. Interestingly, the more disparate the data distributions across clients the more they benefit from ordered learning. We provide analysis explaining this phenomenon, specifically indicating how curriculum training appears to make the objective landscape progressively less convex, suggesting fast converging iterations at the beginning of the training procedure. We derive quantitative results of convergence for both convex and nonconvex objectives by modeling the curriculum training on federated devices as local SGD with locally biased stochastic gradients. Also, inspired by ordered learning, we propose a novel client selection technique that benefits from the real-world disparity in the clients. Our proposed approach to client selection has a synergic effect when applied together with ordered learning in FL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Samarasinghe2023,

title = {CDFSL-V: Cross-Domain Few-Shot Learning for Videos},

author = {Sarinda Samarasinghe and Mamshad Nayeem and Rizve Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CDFSL_Video_Combined_Final.pdf

https://sarinda251.github.io/CDFSL-V-site/

https://www.youtube.com/watch?v=RdlEzfW013o},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Few-shot video action recognition is an effective approach to recognizing new categories with only a few labeled examples, thereby reducing the challenges associated with collecting and annotating large-scale video datasets. Existing methods in video action recognition rely on large labeled datasets from the same domain. However, this setup is not realistic as novel categories may come from different data domains that may have different spatial and temporal characteristics. This dissimilarity between the source and target domains can pose a significant challenge, rendering traditional few-shot action recognition techniques ineffective. To address this issue, in this work, we propose a novel cross-domain few-shot video action recognition method that leverages self-supervised learning and curriculum learning to balance the information from the source and target domains. To be particular, our method employs a masked autoencoder-based self-supervised training objective to learn from both source and target data in a self-supervised manner. Then a progressive curriculum balances learning the discriminative information from the source dataset with the generic information learned from the target domain. Initially, our curriculum utilizes supervised learning to learn class discriminative features from the source data. As the training progresses, we transition to learning target-domain-specific features. We propose a progressive curriculum to encourage the emergence of rich features in the target domain based on class discriminative supervised features in the source domain. %a schedule that helps with this transition. We evaluate our method on several challenging benchmark datasets and demonstrate that our approach outperforms existing cross-domain few-shot learning techniques.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Preserving Modality Structure Improves Multi-Modal Learning },

author = {Swetha Sirnam and Mamshad Nayeem Rizve and Hilde Kuhne and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.13077.pdf

https://arxiv.org/abs/2308.13077

https://github.com/Swetha5/Multi_Sinkhorn_Knopp

https://swetha5.github.io/MultiSK/

https://youtu.be/1CrGkUATy50

},

doi = {https://doi.org/10.48550/arXiv.2308.13077},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Self-supervised learning on large-scale multi-modal datasets allows learning semantically meaningful embeddings in a joint multi-modal representation space without relying on human annotations. These joint embeddings enable zero-shot cross-modal tasks like retrieval and classification. However, these methods often struggle to generalize well on out-of-domain data as they ignore the semantic structure present in modality-specific embeddings. In this context, we propose a novel Semantic-Structure-Preserving Consistency approach to improve generalizability by preserving the modality-specific relationships in the joint embedding space. To capture modality-specific semantic relationships between samples, we propose to learn multiple anchors and represent the multifaceted relationship between samples with respect to their relationship with these anchors. To assign multiple anchors to each sample, we propose a novel Multi-Assignment Sinkhorn-Knopp algorithm. Our experimentation demonstrates that our proposed approach learns semantically meaningful anchors in a self-supervised manner. Furthermore, our evaluation on MSR-VTT and YouCook2 datasets demonstrates that our proposed multi-anchor assignment based solution achieves state-of-the-art performance and generalizes to both inand out-of-domain datasets. Code: https://github.com/Swetha5/Multi_Sinkhorn_Knopp},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023b,

title = {Diffusion Action Segmentation},

author = {Daochang Liu and Qiyue Li and Anh-Dung Dinh and Tingting Jiang and Mubarak Shah and Chang Xu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.17959.pdf

https://arxiv.org/abs/2303.17959

https://finspire13.github.io/DiffAct-Project-Page/

https://github.com/Finspire13/DiffAct

https://youtu.be/o_Jp8shth7U

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Slides.pptx},

doi = { https://doi.org/10.48550/arXiv.2303.17959},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Temporal action segmentation is crucial for understanding long-form videos. Previous works on this task commonly adopt an iterative refinement paradigm by using multi-stage models. We propose a novel framework via denoising diffusion models, which nonetheless shares the same inherent spirit of such iterative refinement. In this framework, action predictions are iteratively generated from random noise with input video features as conditions. To enhance the modeling of three striking characteristics of human actions, including the position prior, the boundary ambiguity, and the relational dependency, we devise a unified masking strategy for the conditioning inputs in our framework. Extensive experiments on three benchmark datasets, i.e., GTEA, 50Salads, and Breakfast, are performed and the proposed method achieves superior or comparable results to state-of-the-art methods, showing the effectiveness of a generative approach for action segmentation.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2023,

title = {TeD-SPAD: Temporal Distinctiveness for Self-supervised Privacy-preservation for video Anomaly Detection},

author = {Joseph Fioresi and Ishan Dave and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.11072.pdf

https://arxiv.org/abs/2308.11072

https://github.com/UCF-CRCV/TeD-SPAD

https://joefioresi718.github.io/TeD-SPAD_webpage/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TeDSPAD_ICCV_poster.pdf

https://youtu.be/3a9qeJUD1GU},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Video-FocalNets: Spatio-Temporal Focal Modulation for Video Action Recognition },

author = {Syed Talal Wasim and Muhammad Uzair Khattak and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.06947.pdf

https://arxiv.org/abs/2307.06947

https://talalwasim.github.io/Video-FocalNets/

https://github.com/TalalWasim/Video-FocalNets

https://talalwasim.github.io/Video-FocalNets/#BibTeX},

doi = { https://doi.org/10.48550/arXiv.2307.06947},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Recent video recognition models utilize Transformer models for long-range spatio-temporal context modeling. Video transformer designs are based on self-attention that can model global context at a high computational cost. In comparison, convolutional designs for videos offer an efficient alternative but lack long-range dependency modeling. Towards achieving the best of both designs, this work proposes Video-FocalNet, an effective and efficient architecture for video recognition that models both local and global contexts. Video-FocalNet is based on a spatio-temporal focal modulation architecture that reverses the interaction and aggregation steps of self-attention for better efficiency. Further, the aggregation step and the interaction step are both implemented using efficient convolution and element-wise multiplication operations that are computationally less expensive than their self-attention counterparts on video representations. We extensively explore the design space of focal modulation-based spatio-temporal context modeling and demonstrate our parallel spatial and temporal encoding design to be the optimal choice. Video-FocalNets perform favorably well against the state-of-the-art transformer-based models for video recognition on three large-scale datasets (Kinetics-400, Kinetics-600, and SS-v2) at a lower computational cost. Our code/models are publicly released.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Efficiently Robustify Pre-Trained Models},

author = {Nishant Jain and Harkirat Behl and Yogesh Singh Rawat and Vibhav Vineet},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV23_Robust_Learning.pdf},

year  = {2023},

date = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {A recent trend in deep learning algorithms has been towards training large scale models, having high parameter count and trained on big dataset. However, robustness of such large scale models towards real-world settings is still a less-explored topic. In this work, we first benchmark the performance of these models under different perturbations and datasets thereby representing real-world shifts, and highlight their degrading performance under these shifts. We then discuss on how complete model fine-tuning based existing robustification schemes might not be a scalable option given very large scale networks and can also lead them to forget some of the desired characteristics. Finally, we propose a simple and cost-effective method to solve this problem, inspired by knowledge transfer literature. It involves robustifying smaller models, at a lower computation cost, and then use them as teachers to tune a fraction of these large scale networks, reducing the overall computational overhead. We evaluate our proposed method under various vision perturbations including ImageNet-C,R,S,A datasets and also for transfer learning, zero-shot evaluation setups on different datasets. Benchmark results show that our method is able to induce robustness to these large scale models efficiently, requiring significantly lower time and also preserves the transfer learning, zero-shot properties of the original model which none of the existing methods are able to achieve. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhou2023,

title = {BT^2 : Backward-compatible Training with Basis Transformation},

author = {Yifei Zhou and Zilu Li and Abhinav Shrivasta and Hengshuang Zhao and Antonio Torralba and Taipeng Tian and Ser-Nam Lim},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2211.03989v3.pdf

https://arxiv.org/abs/2211.03989v3},

doi = {https://doi.org/10.48550/arXiv.2211.03989},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Modern retrieval system often requires recomputing the representation of every piece of data in the gallery when updating to a better representation model. This process is known as backfilling and can be especially costly in the real world where the gallery often contains billions of samples. Recently, researchers have proposed the idea of Backward Compatible Training (BCT) where the new representation model can be trained with an auxiliary loss to make it backward compatible with the old representation. In this way, the new representation can be directly compared with the old representation, in principle avoiding the need for any backfilling. However, followup work shows that there is an inherent tradeoff where a backward compatible representation model cannot simultaneously maintain the performance of the new model itself. This paper reports our ``not-so-surprising'' finding that adding extra dimensions to the representation can help here. However, we also found that naively increasing the dimension of the representation did not work. To deal with this, we propose Backward-compatible Training with a novel Basis Transformation (BT2). A basis transformation (BT) is basically a learnable set of parameters that applies an orthonormal transformation. Such a transformation possesses an important property whereby the original information contained in its input is retained in its output. We show in this paper how a BT can be utilized to add only the necessary amount of additional dimensions. We empirically verify the advantage of BT2 over other state-of-the-art methods in a wide range of settings. We then further extend BT2 to other challenging yet more practical settings, including significant change in model architecture (CNN to Transformers), modality change, and even a series of updates in the model architecture mimicking the evolution of deep learning models.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2023b,

title = {Open-vocabulary Panoptic Segmentation with Embedding Modulation},

author = {Xi Chen and Shuang Li and Ser-Nam Lim and Antonio Torralba and Hengshuang Zhao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.11324.pdf

https://arxiv.org/abs/2303.11324

https://opsnet-page.github.io/},

doi = {https://doi.org/10.48550/arXiv.2303.11324},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Open-vocabulary image segmentation is attracting increasing attention due to its critical applications in the real world. Traditional closed-vocabulary segmentation methods are not able to characterize novel objects, whereas several recent open-vocabulary attempts obtain unsatisfactory results, i.e., notable performance reduction on the closed vocabulary and massive demand for extra data. To this end, we propose OPSNet, an omnipotent and data-efficient framework for Open-vocabulary Panoptic Segmentation. Specifically, the exquisitely designed Embedding Modulation module, together with several meticulous components, enables adequate embedding enhancement and information exchange between the segmentation model and the visual-linguistic well-aligned CLIP encoder, resulting in superior segmentation performance under both open- and closed-vocabulary settings with much fewer need of additional data. Extensive experimental evaluations are conducted across multiple datasets (e.g., COCO, ADE20K, Cityscapes, and PascalContext) under various circumstances, where the proposed OPSNet achieves state-of-the-art results, which demonstrates the effectiveness and generality of the proposed approach. The code and trained models will be made publicly available.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hammoud2023,

title = {Towards a True Evaluation of Rapid Adaptation in Online Continual Learning},

author = {Hasan Abed Al Kader Hammoud and Ameya Prabhu and Ser-Nam Lim and Philip Torr and Adel Bibi and Bernard Ghanem},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2305.09275.pdf

https://arxiv.org/abs/2305.09275

https://github.com/drimpossible/EvalOCL},

doi = {https://doi.org/10.48550/arXiv.2305.09275},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We revisit the common practice of evaluating adaptation of Online Continual Learning (OCL) algorithms through the metric of online accuracy, which measures the accuracy of the model on the immediate next few samples. However, we show that this metric is unreliable, as even vacuous blind classifiers, which do not use input images for prediction, can achieve unrealistically high online accuracy by exploiting spurious label correlations in the data stream. Our study reveals that existing OCL algorithms can also achieve high online accuracy, but perform poorly in retaining useful information, suggesting that they unintentionally learn spurious label correlations. To address this issue, we propose a novel metric for measuring adaptation based on the accuracy on the near-future samples, where spurious correlations are removed. We benchmark existing OCL approaches using our proposed metric on large-scale datasets under various computational budgets and find that better generalization can be achieved by retaining and reusing past seen information. We believe that our proposed metric can aid in the development of truly adaptive OCL methods. We provide code to reproduce our results at https://github.com/drimpossible/EvalOCL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Mitochondria Instance Segmentation with Spatio-Temporal Transformers},

author = {Omkar Thawakar and Rao Muhammad Anwer and Jorma Laaksonen and Orly Reiner and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.12073.pdf

https://github.com/OmkarThawakar/STT-UNET

https://arxiv.org/pdf/2303.12073.pdf

https://link.springer.com/chapter/10.1007/978-3-031-43993-3_59

},

doi = {https://doi.org/10.1007/978-3-031-43993-3_59},

isbn = {978-3-031-43993-3},

year  = {2023},

date = {2023-10-01},

booktitle = {Lecture Notes in Computer Science},

journal = {arXiv:2303.12073},

volume = {14227},

publisher = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},

abstract = {Accurate 3D mitochondria instance segmentation in electron microscopy (EM) is a challenging problem and serves as a prerequisite to empirically analyze their distributions and morphology. Most existing approaches employ 3D convolutions to obtain representative features. However, these convolution-based approaches struggle to effectively capture long-range dependencies in the volume mitochondria data, due to their limited local receptive field. To address this, we propose a hybrid encoder-decoder framework based on a split spatio-temporal attention module that efficiently computes spatial and temporal self-attentions in parallel, which are later fused through a deformable convolution. Further, we introduce a semantic foreground-background adversarial loss during training that aids in delineating the region of mitochondria instances from the background clutter. Our extensive experiments on three benchmarks, Lucchi, MitoEM-R and MitoEM-H, reveal the benefits of the proposed contributions achieving state-of-the-art results on all three datasets. Our code and models are available at https://github.com/OmkarThawakar/STT-UNET.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{deBlegiers2023,

title = {EventTransAct: A Video Transformer-based Framework for Event-camera Based Action Recognition},

author = {Tristan de Blegiers and Ishan Rajendrakumar Dave and Adeel Yousaf and Mubarak Shah},

url = {https://arxiv.org/pdf/2308.13711

https://github.com/tristandb8/EventTransAct

https://tristandb8.github.io/EventTransAct_webpage/

https://www.youtube.com/watch?v=YCff-rTrgco},

year  = {2023},

date = {2023-08-25},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023,

title = {R2Former: Unified retrieval and ranking Transformer for Place Recognition},

author = {Sijie Zhu and Linjie Yang and Chen Chen and Mubarak Shah and Xiaohui Shen and Heng Wang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR_2023_PlaceRecognitionFinal.pdf

https://arxiv.org/pdf/2304.03410.pdf

https://github.com/Jeff-Zilence/R2Former},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Visual Place Recognition (VPR) estimates the location of query images by matching them with images in a reference database. Conventional methods generally adopt aggregated CNN features for global retrieval and RANSAC-based geometric verification for reranking. However, RANSAC only considers geometric information but ignores other possible information that could be useful for reranking, e.g. local feature correlation, and attention values. In this paper, we propose a unified place recognition framework that handles both retrieval and reranking with a novel transformer model, named R2Former. The proposed reranking module takes feature correlation, attention value, and xy coordinates into account, and learns to determine whether the image pair is from the same location. The whole pipeline is end-to-end trainable and the reranking module alone can

also be adopted on other CNN or transformer backbones as a generic component. Remarkably, R2Former significantly

outperforms state-of-the-art methods on major VPR datasets with much less inference time and memory consumption.

It also achieves the state-of-the-art on the holdout MSLS challenge set and could serve as a simple yet strong solution for real-world large-scale applications. Experiments also show vision transformer tokens are comparable and sometimes better than CNN local features on local matching. The code will be publicly available. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023b,

title = {Class Prototypes based Contrastive Learning for Classifying Multi-Label and Fine-Grained Educational Videos},

author = {Rohit Gupta and Anirban Roy and Sujeong Kim and Claire Christensen and Todd Grindal and Sarah Nixon Gerard and Madeline Cincebeaux and Ajay Divakaran and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Rohit_SRI_CVPR2023_Multi_Modal_Multi_Label_Contrastive_Learning_Camera_Ready-4.pdf

https://www.rohitg.xyz/MMContrast/

https://nusci.csl.sri.com/project/APPROVE},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The recent growth in the consumption of online media by children during early childhood necessitates data-driven tools enabling educators to filter out appropriate educational content for young learners. This paper presents an approach for detecting educational content in online videos. We focus on two widely used educational content classes: literacy and math. For each class, we choose prominent codes (sub-classes) based on the Common Core Standards. For example, literacy codes include ‘letter names’, ‘letter sounds’, and math codes include ‘counting’, ‘sorting’. We pose this as a fine-grained multilabel classification problem as videos can contain multiple types of educational content and the content classes can get visually similar (e.g., ‘letter names’ vs ‘letter sounds’). We propose a novel class prototypes based supervised contrastive learning approach that can handle fine-grained samples associated with multiple labels. We learn a class prototype for each class and a loss function is employed to minimize the distances between a class prototype and the samples from the class. Similarly,

distances between a class prototype and the samples from other classes are maximized. As the alignment between visual

and audio cues are crucial for effective comprehension, we consider a multimodal transformer network to capture the interaction between visual and audio cues in videos while learning the embedding for videos. For evaluation, we present a dataset, APPROVE, employing educational videos from YouTube labeled with fine-grained education classes by education researchers. APPROVE consists of 193 hours of expert-annotated videos with 19 classes. The proposed approach outperforms strong baselines on APPROVE and other benchmarks such as Youtube-8M, and COIN. The dataset is available at https://nusci.csl.sri.com/project/APPROVE.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2023,

title = {TimeBalance: Temporally-Invariant and Temporally-Distinctive Video Representations for Semi-Supervised Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TimeBalance_CVPR23_arxiv.pdf

https://daveishan.github.io/timebalance_webpage/

https://github.com/DAVEISHAN/TimeBalance},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Semi-Supervised Learning can be more beneficial for the video domain compared to images because of its higher annotation

cost and dimensionality. Besides, any video understanding task requires reasoning over both spatial and temporal dimensions. In order to learn both the static and motion related features for the semi-supervised action recognition task, existing methods rely on hard input inductive biases like using two-modalities (RGB and Optical-flow) or two-stream of different playback rates.

Instead of utilizing unlabeled videos through diverse input streams, we rely on self-supervised video representations,

particularly, we utilize temporally-invariant and temporally-distinctive representations. We observe that these representations complement each other depending on the nature of the action. Based on this observation, we propose a student-teacher semi-supervised learning framework, TimeBalance, where we distill the knowledge from a temporally-invariant and a temporally-distinctive teacher. Depending on the nature of the unlabeled video, we dynamically combine the knowledge of these two  teachers based on a novel temporal similarity-based reweighting scheme. Our method achieves state-of-the-art performance

on three action recognition benchmarks: UCF101, HMDB51, and Kinetics400. Code: https://github.com/DAVEISHAN/TimeBalance.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2023,

title = {PivoTAL: Prior-Driven Supervision for Weakly-Supervised Temporal Action Localization},

author = {Mamshad Nayeem Rizve and Gaurav Mittal and Ye Yu and Matthew Hall and Sandra Sajeev and Mubarak Shah and Mei Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023_Supplemental_Material.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR2023_Poster.pdf

https://www.youtube.com/watch?v=6kAoQjXfzio},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Urooj2023,

title = {Learning Situation Hyper-Graphs for Video Question Answering},

author = {Aisha Urooj and Hilde Kuehne and Bo Wu and Kim Chheu and Walid Bousselham and Chuang Gan and Niels Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2023072364-4.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SHG_VQA_CVPR2023_cam_ready_supp.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Answering questions about complex situations in videos requires not only capturing the presence of actors, objects, and their relations but also the evolution of these relationships over time. A situation hyper-graph is a representation that describes situations as scene sub-graphs for video frames and hyper-edges for connected sub-graphs and has been proposed to capture all such information in a compact structured form. In this work, we propose an architecture for Video Question Answering (VQA) that enables answering questions related to video content by predicting situation hyper-graphs, coined Situation Hyper-Graph based Video Question Answering (SHG-VQA). To this end, we train a situation hyper-graph decoder to implicitly identify graph representations with actions and object/human-object relationships from the input video clip. and to use cross-attention

between the predicted situation hyper-graphs and the question embedding to predict the correct answer. The proposed

method is trained in an end-to-end manner and optimized by a VQA loss with the cross-entropy function and a Hungarian

matching loss for the situation graph prediction. The effectiveness of the proposed architecture is extensively evaluated

on two challenging benchmarks: AGQA and STAR. Our results show that learning the underlying situation hypergraphs

helps the system to significantly improve its performance for novel challenges of video question-answering tasks. },

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bhunia2023,

title = {Person Image Synthesis via Denoising Diffusion Model},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Jorma Tapio Laaksonen and Mubarak Shah and Fahad Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/person_image_synthesis_via_den-Camera-ready-PDF.pdf

https://lnkd.in/d-8v3r8B

https://lnkd.in/dGPTjvge

https://lnkd.in/dxcGQsUX

https://github.com/ankanbhunia/PIDM},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The pose-guided person image generation task requires synthesizing photorealistic images of humans in arbitrary poses. The existing approaches use generative adversarial networks that do not necessarily maintain realistic textures or need dense correspondences that struggle to handle complex  deformations and severe occlusions. In this work, we show how denoising diffusion models can be applied for high-fidelity person image synthesis with

strong sample diversity and enhanced mode coverage of the learnt data distribution. Our proposed Person Image Diffusion Model (PIDM) disintegrates the complex transfer problem into a series of simpler forward-backward denoising steps. This helps in learning plausible source-to-target transformation trajectories that result in faithful textures and undistorted appearance details. We introduce a ‘texture diffusion module’ based on cross-attention to accurately model the correspondences between appearance and pose information available in source and target images. Further, we propose  disentangled classifier-free guidance’ to ensure close resemblance between the conditional inputs and the synthesized output in terms of both pose and appearance information. Our extensive results on two large-scale benchmarks and a user study demonstrate the photorealism of our proposed approach under challenging scenarios. We also show how our generated images can help in downstream tasks. Code is available at https://github.com/ankanbhunia/PIDM.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wasim2023,

title = {Vita-CLIP: Video and text adaptive CLIP via Multimodal Prompting},

author = {Syed Talal Wasim and Muzammal Naseer and Salman Khan and Fahad Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/vita_clip_video_and_text_adapt-Camera-ready-PDF.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Adopting contrastive image-text pretrained models like CLIP towards video classification has gained attention due to its cost-effectiveness and competitive performance. However, recent works in this area face a trade-off. Finetuning the pretrained model to achieve strong supervised performance results in low zero-shot generalization. Similarly, freezing the backbone to retain zero-shot capability causes significant drop in supervised accuracy. Because of this,

recent works in literature typically train separate models for supervised and zero-shot action recognition. In this work, we propose a multimodal prompt learning scheme that works to balance the supervised and zero-shot performance under a single unified training. Our prompting approach on the vision side caters for three aspects: 1) Global video-level prompts to model the data distribution; 2) Local frame-level prompts to provide per-frame discriminative

conditioning; and 3) a summary prompt to extract a condensed video representation. Additionally, we define a prompting scheme on the text side to augment the textual context. Through this prompting scheme, we can achieve state-of-the-art zero-shot performance on Kinetics-600, HMDB51 and UCF101 while remaining competitive in the supervised setting. By keeping the pretrained backbone frozen, we optimize a much lower number of parameters and retain the existing general representation which helps achieve the strong zero-shot performance. Our codes and models will be publicly released. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Clark2023,

title = {Where We Are and What We're Looking At: Query Based Worldwide Image Geo-localization Using Hierarchies and Scenes},

author = {Brandon Eric Clark and Alec Kerrigan and Parth Parag Kulkarni and Vicente Vivanco Cepeda and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Camera-Ready-Full-Paper.pdf

https://github.com/AHKerrigan/GeoGuessNet

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR23-Poster_THU-PM-246-1.pdf

https://www.youtube.com/watch?v=fp3hZGbwPqk},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Determining the exact latitude and longitude that a photo was taken is a useful and widely applicable task, yet it remains exceptionally difficult despite the accelerated progress of other computer vision tasks. Most previous approaches have opted to learn single representations of query images, which are then classified at different levels of geographic granularity. These approaches fail to exploit the different visual cues that give context to different hierarchies, such as the country, state, and city level. To this end, we introduce an end-to-end transformer-based architecture that exploits the relationship between different geographic levels (which we refer to as hierarchies) and the corresponding visual scene information in an image through hierarchical cross-attention. We achieve this by learning a query for each geographic hierarchy and scene type. Furthermore, we learn a separate representation for different environmental scenes, as different scenes in the same location are often defined by completely different visual features. We achieve state of the art accuracy on 4 standard geo-localization datasets : Im2GPS, Im2GPS3k, YFCC4k, and YFCC26k, as well as qualitatively demonstrate how our method learns different representations for different visual hierarchies and scenes, which has not been demonstrated in the previous methods. Above previous testing  datasets mostly consist of iconic landmarks or images taken from social media, which makes the dataset a simple memory task, or makes it biased towards certain places. To address this issue we introduce a much harder testing dataset, Google-World-Streets-15k, comprised of images taken from Google Streetview covering the whole planet and present state of the art results. Our code can be found at https://github.com/AHKerrigan/GeoGuessNet.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2023,

title = {Hybrid Active Learning via Deep Clustering for Video Action Detection},

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/hybrid-active-learning-via-deep-clustering-for-video-action-detection/},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chantry2023,

title = {A Large-scale Robustness Analysis of Video Action Recognition Models},

author = {Madeline Chantry and Naman Biyani and Prudvi Kamtam and Shruti Vyas and Hamid Palangi and Vibhav Vineet and Yogesh Rawat},

url = {https://sites.google.com/view/videorobustnessbenchmark/home

https://www.crcv.ucf.edu/research/projects/ucf101-ds-action-recognition-for-real-world-distribution-shifts/

https://github.com/Maddy12/ActionRecognitionRobustnessEval

https://youtu.be/pv2AJ_t-v90

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PosterCVPR2023.png},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023b,

title = {TopNet: Transformer-based Object Placement Network for Image Compositing },

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://arxiv.org/pdf/2304.03372.pdf},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023,

title = {FeatER: An Efficient Network for Human Reconstruction via Feature Map-Based TransformER},

author = {Ce Zheng and Matias Mendieta and Taojiannan Yang and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2205.15448.pdf

https://zczcwh.github.io/feater_page/

https://github.com/zczcwh/FeatER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023b,

title = {POTTER: Pooling Attention Transformer for Efficient Human Mesh Recovery},

author = {Ce Zheng and Xianpeng Liu and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2303.13357.pdf

https://zczcwh.github.io/potter_page/

https://github.com/zczcwh/POTTER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PoseFormerV2: Exploring Frequency Domain for Efficient and Robust 3D Human Pose Estimation},

author = {Qitao Zhao and Ce Zheng and Mengyuan Liu and Pichao Wang and Chen Chen},

url = {https://arxiv.org/pdf/2303.17472.pdf

https://qitaozhao.github.io/PoseFormerV2

https://github.com/QitaoZhao/PoseFormerV2},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2023,

title = {On the Robustness of Large Multimodal Models Against Image Adversarial Attacks},

author = {Xuanming Cui and Alejandro Aparcedo and Young Kyun Jang and Ser-Nam Lim},

url = {https://arxiv.org/pdf/2312.03777

https://arxiv.org/abs/2312.03777},

doi = {https://doi.org/10.48550/arXiv.2312.03777},

year  = {2023},

date = {2023-06-18},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference},

abstract = {Recent advances in instruction tuning have led to the development of State-of-the-Art Large Multimodal Models (LMMs). Given the novelty of these models, the impact of visual adversarial attacks on LMMs has not been thoroughly examined. We conduct a comprehensive study of the robustness of various LMMs against different adversarial attacks, evaluated across tasks including image classification, image captioning, and Visual Question Answer (VQA). We find that in general LMMs are not robust to visual adversarial inputs. However, our findings suggest that context provided to the model via prompts, such as questions in a QA pair helps to mitigate the effects of visual adversarial inputs. Notably, the LMMs evaluated demonstrated remarkable resilience to such attacks on the ScienceQA task with only an 8.10% drop in performance compared to their visual counterparts which dropped 99.73%. We also propose a new approach to real-world image classification which we term query decomposition. By incorporating existence queries into our input prompt we observe diminished attack effectiveness and improvements in image classification accuracy. This research highlights a previously under-explored facet of LMM robustness and sets the stage for future work aimed at strengthening the resilience of multimodal systems in adversarial environments.},

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2023,

title = {3DMODT: Attention-Guided Affinities for Joint Detection & Tracking in 3D Point Clouds},

author = {Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/pdf/2211.00746.pdf},

year  = {2023},

date = {2023-05-29},

urldate = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sangam2023,

title = {TransVisDrone: Spatio-Temporal Transformer for Vision-based Drone-to-Drone Detection in Aerial Videos},

author = {Tushar Sangam and Ishan Rajendrakumar Dave and Waqas Sultani and Mubarak Shah},

url = {https://arxiv.org/pdf/2210.08423.pdf

https://tusharsangam.github.io/TransVisDrone-project-page/},

year  = {2023},

date = {2023-05-29},

urldate = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2023,

title = {AIM: Adapting Image Models for Efficient Video Understanding},

author = {Taojiannan Yang and Yi Zhu and Yusheng Xie and Aston Zhang and Chen Chen and Mu Li},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {AIM: Adapting Image Models for Efficient Video Understanding},

publisher = {Eleventh International Conference on Learning Representations (ICLR)},

abstract = {Recent vision transformer based video models mostly follow the ``image pre-training then finetuning" paradigm and have achieved great success on multiple video benchmarks. However, full finetuning such a video model could be computationally expensive and unnecessary, given the pre-trained image transformer models have demonstrated exceptional transferability. In this work, we propose a novel method to Adapt pre-trained Image Models (AIM) for efficient video understanding. By freezing the pre-trained image model and adding a few lightweight Adapters, we introduce spatial adaptation, temporal adaptation and joint adaptation to gradually equip an image model with spatiotemporal reasoning capability. We show that our proposed AIM can achieve competitive or even better performance than prior arts with substantially fewer tunable parameters on four video action recognition benchmarks. Thanks to its simplicity, our method is also generally applicable to different image pre-trained models, which has the potential to leverage more powerful image foundation models in the future. },

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Re-calibrating Feature Attributions for Model Interpretation},

author = {Peiyu Yang and Naveed Akhtar and Zeyi Wen and Mubarak Shah and Ajmal Mian},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {Re-calibrating Feature Attributions for Model Interpretation},

publisher = {Eleventh International Conference on Learning Representations (ICLR), notable top 25%},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Beetham2023b,

title = {Dual Student Networks for Data-Free Model Stealing},

author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/abs/2309.10058},

year  = {2023},

date = {2023-05-01},

booktitle = {Eleventh International Conference on Learning Representations},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023,

title = {Efficient Distribution Similarity Identification in Clustered Federated Learning via Principal Angles Between Client Data Subspaces},

author = {Saeed Vahidian and Mahdi Morafah and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://arxiv.org/abs/2209.10526},

year  = {2023},

date = {2023-02-07},

urldate = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhong2023,

title = {Refined Semantic Enhancement Towards Frequency Diffusion for Video Captioning},

author = {Xian Zhong and Zipeng Li and Shuqin Chen and Kui Jiang and Chen Chen and Mang Ye},

url = {https://arxiv.org/abs/2211.15076},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023,

title = {Novel Motion Patterns Matter for Practical Skeleton-based Action Recognition},

author = {Mengyuan Liu and Fanyang Meng and Chen Chen and Songtao Wu},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023,

title = {Contrastive Self-Supervised Learning Leads to Higher Adversarial Susceptibility},

author = {Rohit Gupta and Naveed Akhtar and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2207.10862.pdf},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Are all Frames Equal? Active Sparse Labeling for Video Action Detection },

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/active-sparse-labeling-for-video-action-detection/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/neurips_poster_ASL_upload.png

https://github.com/aayushjr/ASL-video },

year  = {2022},

date = {2022-11-28},

urldate = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Video action detection requires annotations at every frame, which drastically increases the labeling cost. In this work, we focus on efficient labeling of videos for action detection to minimize this cost. We propose active sparse labeling (ASL), a novel active learning strategy for video action detection. We propose a novel frame-level scoring mechanism aimed at selecting the most informative frames in a video. We also introduce a novel loss formulation which enables training of action detection model with these sparsely selected frames. We evaluated the proposed approach on two different action detection benchmark datasets, UCF-101-24 and J-HMDB-21, and observed that active sparse labeling can be very effective in saving annotation costs. We demonstrate that the proposed approach performs better than random selection, outperforming all other baselines, with performance comparable to supervised approach using merely 10% annotations.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schiappa2022,

title = {Robustness Analysis of Video-Language Models Against Visual and Language Perturbations},

author = {Madeline Chantry Schiappa and Shruti Vyas and Hamid Palangi and Yogesh Rawat and Vibhav Vineet},

url = {https://sites.google.com/view/videolanguagerobustness/home

https://openreview.net/forum?id=A79jAS4MeW9

https://github.com/Maddy12/VideoLanguageModelRobustness/tree/master},

year  = {2022},

date = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Joint visual and language modeling on large-scale datasets has recently shown good progress in multi-modal tasks when compared to single modal learning. However, robustness of these  approaches against real-world perturbations has not been studied. In this work, we perform the first extensive robustness study of video-language models against various real-world perturbations. We focus on text-to-video retrieval and propose two large-scale benchmark datasets, MSRVTT-P and YouCook2-P, which utilize 90 different visual and 35 different text perturbations. The study reveals some interesting initial findings from the studied models: 1) models are more robust when text is perturbed versus when video is perturbed, 2) models that are pre-trained are more robust than those trained from scratch, 3) models attend more to scene and objects rather than motion and action. We hope this study will serve as a benchmark and guide future research in robust video-language learning. The benchmark introduced in this study along with the code and datasets is available at https://bit.ly/3CNOly4.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xu2022,

title = {Don’t Pour Cereal into Coffee: Differentiable Temporal Logic for Temporal Action Segmentation},

author = {Ziwei Xu and Yogesh Rawat and Yongkang Wong and Mohan Kankanhalli and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ziwei_neurips2022.pdf

https://diff-tl.github.io/

https://github.com/ZiweiXU/DTL-action-segmentation},

year  = {2022},

date = {2022-11-09},

urldate = {2022-11-09},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {We propose Differentiable Temporal Logic (DTL), a model-agnostic framework that introduces temporal constraints to deep networks. DTL treats the outputs of a network as a truth assignment of a temporal logic formula, and computes a temporal logic loss reflecting the consistency between the output and the constraints. We propose a comprehensive set of constraints, which are implicit in data annotations, and incorporate them with deep networks via DTL. We evaluate the effectiveness of DTL on the temporal action segmentation task and observe improved performance and reduced logical errors in the output of different task models. Furthermore, we provide an extensive analysis to visualize the desirable effects of DTL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2022,

title = {GAMa: Cross-view Video Geo-localization},

author = {Shruti Vyas and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512-supp.pdf

https://youtu.be/KSHuer_VXJo},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {The existing work in cross-view geo-localization is based on images where a ground panorama is matched to an aerial image. In this work, we focus on ground videos instead of images which provides ad-ditional contextual cues which are important for this task. There are no existing datasets for this problem, therefore we propose GAMa dataset, a large-scale dataset with ground videos and corresponding aerial images. We also propose a novel approach to solve this problem. At clip-level, a short video clip is matched with corresponding aerial image and is later used to get video-level geo-localization of a long video. Moreover, we propose a hierarchical approach to further improve the clip-level geo-localization. On this challenging dataset, with unaligned images and lim-ited field of view, our proposed method achieves a Top-1 recall rate of 19.4% and 45.1% @1.0mile. Code & dataset are available at this link.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2022,

title = {Med-DANet: Dynamic Architecture Network for Efficient Medical Volumetric Segmentation},

author = {Wenxuan Wang and Chen Chen and Jing Wang and Sen Zha and Yan Zhang and Jiangyun Li},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2206.06575.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {For 3D medical image (e.g. CT and MRI) segmentation, the difficulty of segmenting each slice in a clinical case varies greatly. Previous research on volumetric medical image segmentation in a slice-byslice manner conventionally use the identical 2D deep neural network to segment all the slices of the same case, ignoring the data heterogeneity among image slices. In this paper, we focus on multi-modal 3D MRI brain tumor segmentation and propose a dynamic architecture network named Med-DANet based on adaptive model selection to achieve effective accuracy and efficiency trade-off. For each slice of the input 3D MRI volume, our proposed method learns a slice-specific decision by the Decision Network to dynamically select a suitable model from the predefined Model Bank for the subsequent 2D segmentation task. Extensive experimental results on both BraTS 2019 and 2020 datasets show that our proposed method achieves comparable or better results than previous state-of-the art methods for 3D MRI brain tumor segmentation with much less model complexity. Compared with the state-of-the-art 3D method TransBTS, the proposed framework improves the model efficiency by up to 3.5 × without sacrificing the accuracy. Our code will be publicly available at https://github.com/Wenxuan-1119/Med-DANet.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2022,

title = {GALA: Toward Geometry-and-Lighting-Aware Object Search for Compositing},

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.00125.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GALA_supplementary.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Compositing-aware object search aims to find the most compatible objects for compositing given a background image and a query bounding box. Previous works focus on learning compatibility between the foreground object and background, but fail to learn other important factors from large-scale data, i.e. geometry and lighting. To move a step further, this paper proposes GALA (Geometry-and-Lighting-Aware), a generic foreground object search method with discriminative modeling on geometry and lighting compatibility for open-world image compositing. Remarkably, it achieves state-of-the-art results on the CAIS dataset and generalizes well on large-scale open-world datasets, i.e. Pixabay and

Open Images. In addition, our method can effectively handle non-box scenarios, where users only provide background images without any input bounding box. A web demo (see supplementary materials) is built to showcase applications of the proposed method for compositing-aware search and automatic location/scale prediction for the foreground object. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2022,

title = {Weakly Supervised Grounding for VQA in Vision-Language Transformers},

author = {Aisha Urooj Khan and Hilde Kuehne and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011-supp.pdf

https://github.com/aurooj/WSG-VQA-VLTransformers

https://youtu.be/dekmVb6lq3I},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Transformers for visual-language representation learning have been getting a lot of interest and shown tremendous performance on visual question answering (VQA) and grounding. However, most systems that show good performance of those tasks still rely on pre-trained object

detectors during training, which limits their applicability to the object classes available for those detectors. To mitigate this limitation, this paper

focuses on the problem of weakly supervised grounding in the context of visual question answering in transformers. Our approach leverages

capsules by transforming each visual token into a capsule representation in the visual encoder; it then uses activations from language self-attention layers as a text-guided selection module to mask those capsules before they are forwarded to the next layer. We evaluate our approach on the challenging GQA as well as VQA-HAT dataset for VQA grounding. Our experiments show that: while removing the information of masked

objects from standard transformer architectures leads to a significant drop in performance, the integration of capsules significantly improves the grounding ability of such systems and provides new state-of-the-art results compared to other approaches in the field.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022,

title = {OpenLDN: Learning to Discover Novel Classes for Open-World Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665-supp.pdf

https://github.com/nayeemrizve/OpenLDN

https://youtu.be/p2lYqvklcjA},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Semi-supervised learning (SSL) is one of the dominant approaches to address the annotation bottleneck of supervised learning. Recent SSL methods can effectively leverage a large repository of unlabeled data to improve performance while relying on a small set of labeled data. One common assumption in most SSL methods is that the labeled and unlabeled data are from the same data distribution. However, this is hardly the case in many real-world scenarios, which limits their applicability. In this work, instead, we attempt to solve the challenging open-world SSL problem that does not make such an assumption. In the open-world SSL problem, the objective is to recognize samples of known classes, and simultaneously detect and cluster samples belonging to novel classes present in unlabeled data. This work introduces OpenLDN that utilizes a pairwise similarity loss to discover novel classes. Using a bi-level optimization rule this pairwise similarity loss exploits the information available in the labeled set to implicitly cluster novel class samples, while simultaneously recognizing samples from known classes. After discovering novel  classes, OpenLDN transforms the open-world SSL problem into a standard SSL problem to achieve additional performance gains using existing SSL methods. Our extensive experiments demonstrate that OpenLDN outperforms the current state-of-the-art methods on multiple popular classification benchmarks while providing a better accuracy/training time trade-off. Code: https://github.com/nayeemrizve/OpenLDN},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022b,

title = {Towards Realistic Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402-supp.pdf

https://github.com/nayeemrizve/TRSSL

https://youtu.be/mE7GeQ35WyY},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Deep learning is pushing the state-of-the-art in many computer

vision applications. However, it relies on large annotated data

repositories, and capturing the unconstrained nature of the real-world

data is yet to be solved. Semi-supervised learning (SSL) complements

the annotated training data with a large corpus of unlabeled data to

reduce annotation cost. The standard SSL approach assumes unlabeled

data are from the same distribution as annotated data. Recently, a more

realistic SSL problem, called open-world SSL, is introduced, where the

unannotated data might contain samples from unknown classes. In this paper, we propose a novel pseudo-label based approach to tackle SSL in

open-world setting. At the core of our method, we utilize sample uncertainty and incorporate prior knowledge about class distribution to  generate reliable class-distribution-aware pseudo-labels for unlabeled data belonging to both known and unknown classes. Our extensive experimentation showcases the effectiveness of our approach on several benchmark datasets, where it substantially outperforms the existing state-of-the art on seven diverse datasets including CIFAR-100 (∼17%), ImageNet-100 (∼5%), and Tiny ImageNet (∼9%). We also highlight the flexibility of our approach in solving novel class discovery task, demonstrate its stability in dealing with imbalanced data, and complement our approach with a technique to estimate the number of novel classes. Code: https://github.com/nayeemrizve/TRSSL},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2022,

title = {Self Supervised Learning for Multiple Object Tracking in 3D Point Clouds},

author = {Aakash Kumar and Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/camera_ready_paper.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {2022 IEEE/RSJ International Conference on Intelligent Robots and Systems},

abstract = {Multiple object tracking in 3D point clouds has applications in mobile robots and autonomous driving. This is a challenging problem due to the sparse nature of the point clouds and the added difficulty of annotation in 3D for supervised learning. To overcome these challenges, we propose a neural network architecture that learns effective object features and their affinities in a self supervised fashion for multiple object tracking in 3D point clouds captured with LiDAR sensors. For self supervision, we use two approaches. First, we generate two augmented LiDAR frames from a single real frame by applying translation, rotation and cutout to the objects. Second, we synthesize a LiDAR frame using CAD models or primitive geometric shapes and then apply the above three augmentations to them. Hence, the ground truth object locations and associations are known in both frames for self supervision. This removes the need to annotate object associations in real data, and additionally the need for training data collection and annotation for object detection in synthetic data. To the best of our knowledge, this is the first self supervised multiple object tracking method for 3D data. Our model achieves state of the art results.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Arif2022,

title = {Background-Tolerant Object Classification with Embedded Segmentation Mask for Infrared and Color Imagery},

author = {Maliha Arif and Calvin Yong and Abhijit Mahalanobis and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Final_ICIP2022_MA_submission.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/bg_poster_ICIP2022_Single.jpg},

year  = {2022},

date = {2022-10-16},

urldate = {2022-10-16},

booktitle = {IEEE International Conference on Image Processing},

abstract = {Even though convolutional neural networks (CNNs) can classify objects in images very accurately, it is well known that the attention of the network may not always be on the semantically important regions of the scene. It has been observed that networks often learn background textures, which are not relevant to the object of interest. In turn this makes the networks susceptible to variations and changes in the background

which may negatively affect their performance. 



We propose a new three-step training procedure called split training to reduce this bias in CNNs for object recognition using Infrared imagery and Color (RGB) data. Our split training procedure has three steps. First, a baseline model is trained to recognize objects in images without background, and the activations produced by the higher layers are observed. Next, a second network is trained using Mean Square Error (MSE)  loss to produce the same activations, but in response to the objects embedded in background. This forces the second network to ignore the  background while focusing on the object of interest. Finally, with layers producing the activations frozen, the rest of the second network is  trained using cross-entropy loss to classify the objects in images with background. Our training method outperforms the traditional training procedure in both a simple CNN architecture, as well as for deep CNNs like VGG and DenseNet, and learns to mimic human vision which focuses more on shape and structure than background with higher accuracy. 



Index Terms— infrared imagery, background invariant learning, grad-CAM, split training, MS-COCO},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2023,

title = {DEEPSAR: Vessel Detection In SAR Imagery With Noisy Labels},

author = {Manu S Pillai and Abhijeet Bhattacharya and Tanmay Baweja and Rohit Gupta and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICIP_Submission.pdf},

year  = {2022},

date = {2022-10-16},

urldate = {2023-10-08},

publisher = {IEEE International Conference on Image Processing},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2022b,

title = {Tag-Based Attention Guided Bottom-Up Approach for Video Instance Segmentation},

author = {Jyoti Kini and Mubarak Shah },

url = {https://arxiv.org/pdf/2204.10765.pdf},

year  = {2022},

date = {2022-08-21},

urldate = {2022-08-21},

booktitle = {26th International Conference on Pattern Recognition},

issue = {arxiv:2204.10765},

keywords = {ICPR, Video Instance Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Beetham2022,

title = {Detecting Compromised Architecture/Weights of a Deep Model},

author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},

url = {https://ieeexplore.ieee.org/abstract/document/9956280},

year  = {2022},

date = {2022-08-21},

urldate = {2022-08-21},

booktitle = {26th International Conference on Pattern Recognition},

keywords = {ICPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Self-Supervised Predictive Convolutional Attentive Block for Anomaly Detection},

author = {Nicolae-Catalin Ristea and Neelu Madan and Radu Tudor Ionescu and Kamal Nasrollahi and Fahad Shahbaz Khan and Thomas B. Moeslund and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SSPCAB_camera-arxiv.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Self-Supervised Learning, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UNICON: Combating Label Noise Through Uniform Selection and Contrastive Learning},

author = {Nazmul Karim and Mamshad Nayeem Rizve and Nazanin Rahnavard and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363-supp.pdf

https://github.com/nazmul-karim170/unicon-noisy-label},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Contrastive Learning, CVPR, Noisy Labels, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UBnormal: New Benchmark for Supervised Open-Set Video Anomaly Detection},

author = {Andra Acsintoae and Andrei Florescu and Mariana-Iuliana Georgescu and Tudor Mare and Paul Sumedrea and Radu Tudor Ionescu and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/04315.pdf

https://github.com/lilygeorgescu/UBnormal},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Dataset},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {SPAct: Self-supervised Privacy Preservation for Action Recognition},

author = {Ishan Rajendrakumar Dave and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2203.15205.pdf

https://github.com/DAVEISHAN/SPAct

https://www.youtube.com/watch?v=_PAlMT7ozts},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Action Recognition, CVPR, Privacy Preservation},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {End-to-End Semi-Supervised Learning for Video Action Detection},

author = {Akash Kumar and Yogesh Singh Rawat},

url = {https://arxiv.org/pdf/2203.04251.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Local Learning Matters: Rethinking Data Heterogeneity in Federated Learning},

author = {Matias Mendieta and Taojiannan Yang and Pu Wang and Minwoo Lee and Zhengming Ding and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405_supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Federated Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},

author = {Sijie Zhu and Mubarak Shah and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Cross-View, CVPR, Geo-Localization, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PSTR: End-to-End One-Step Person Search With Transformers},

author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf

https://github.com/JialeCao001/PSTR},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for

discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step

transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging

PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},

keywords = {CVPR, Re-Identification, Transformers, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {OW-DETR: Open-world Detection Transformer},

author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf

https://github.com/akshitac8/OW-DETR.},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while  simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components

namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental

object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},

keywords = {CVPR, Object Detection, Open World, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2022,

title = {Video Action Detection: Analysing Limitations and Challenges},

author = {Rajat Modi and Aayush Jung Rana and Akash Kumar and Praveen Tirupattur and Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.07892.pdf},

year  = {2022},

date = {2022-04-17},

urldate = {2022-04-17},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Performance Evaluation of Boosted 2-stream TCRNet},

author = {Shah Hassan and MdJibanul Haque Jiban and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/paper.pdf},

year  = {2022},

date = {2022-03-02},

urldate = {2022-03-02},

publisher = {International Congress on Information and Communication Technology},

keywords = {ICICT},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pestana2022,

title = {Transferable 3D Adversarial Textures using End-to-end Optimization},

author = {Camilo Pestana and Naveed Akhtar and Nazanin Rahnavard and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/WACV_2022_Deceptive_Textures-1.pdf},

doi = {10.1109/WACV51458.2022.00080},

year  = {2022},

date = {2022-02-15},

urldate = {2022-02-15},

pages = {727-736},

publisher = {IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},

abstract = {Deep visual models are known to be vulnerable to adversarial attacks. The last few years have seen numerous techniques to compute adversarial inputs for these models. However, there are still under-explored avenues in this critical research direction. Among those is the estimation of adversarial textures for 3D models in an end-to-end optimization scheme. In this paper, we propose such a scheme to generate adversarial textures for 3D models that are highly transferable and invariant to different camera views and lighting conditions. Our method makes use of neural rendering with explicit control over the model texture and background. We ensure transferability of the adversarial textures by employing an ensemble of robust and non-robust models. Our technique utilizes 3D models as a proxy to simulate closer to real-life conditions, in contrast to conventional use of 2D images for adversarial attacks. We show the efficacy of our method with extensive experiments.},

keywords = {Adversarial Attacks, WACV},

pubstate = {published},

tppubtype = {conference}

}