Mubarak Shah

@conference{Vayani2025,

title = {All Languages Matter: Evaluating LMMs on Culturally Diverse 100 Languages},

author = {Ashmal Vayani and Dinura Dissanayake and Hasindri Watawana and Noor Ahsan and Nevasini Sasikumar and Omkar Thawakar and Henok Biadglign Ademtew and Yahya Hmaiti and Amandeep Kumar and Kartik Kuckreja and Mykola Maslych and Wafa Al Ghallabi and Mihail Mihaylov and Chao Qin and Abdelrahman M Shaker and Mike Zhang and Mahardika Krisna Ihsani and Amiel Esplana and Monil Gokani and Shachar Mirkin and Harsh Singh and Ashay Srivastava and Endre Hamerlik and Fathinah Asma Izzati and Fadillah Adamsyah Maani and Sebastian Cavada and Jenny Chim and Rohit Gupta and Sanjay Manjunath and Kamila Zhumakhanova and Feno Heriniaina Rabevohitra and Azril Amirudin and Muhammad Ridzuan and Daniya Kareem and Ketan More and Kunyang Li and Pramesh Shakya and Muhammad Saad and Amirpouya Ghasemaghaei and Amirbek Djanibekov and Dilshod Azizov and Branislava Jankovic and Naman Bhatia and Alvaro Cabrera and Johan Obando-Ceron and Olympiah Otieno and Fabian Farestam and Muztoba Rabbani and Sanoojan Baliah and Santosh Sanjeev and Abduragim Shtanchaev and Maheen Fatima and Thao Nguyen and Amrin Kareem and Toluwani Aremu and Nathan Xavier and Amit Bhatkal and Hawau Toyin and Aman Chadha and Hisham Cholakkal and Rao Muhammad Anwer and Michael Felsberg and Jorma Laaksonen and Thamar Solorio and Monojit Choudhury and Ivan Laptev and Mubarak Shah and Salman Khan and Fahad Khan},

url = {https://arxiv.org/abs/2411.16508

https://mbzuai-oryx.github.io/ALM-Bench/

https://huggingface.co/datasets/MBZUAI/ALM-Bench

https://github.com/mbzuai-oryx/ALM-Bench},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Existing Large Multimodal Models (LMMs) generally focus on only a few regions and languages. As LMMs continue to improve, it is increasingly important to ensure they understand cultural contexts, respect local sensitivities, and support low-resource languages, all while effectively integrating corresponding visual cues. In pursuit of culturally diverse global multimodal models, our proposed All Languages Matter Benchmark (ALM-bench) represents the largest and most comprehensive effort to date for evaluating LMMs across 100 languages. ALM-bench challenges existing models by testing their ability to understand and reason about culturally diverse images paired with text in various languages, including many low-resource languages traditionally underrepresented in LMM research. The benchmark offers a robust and nuanced evaluation framework featuring various question formats, including true/false, multiple choice, and open-ended questions, which are further divided into short and long-answer categories. ALM-bench design ensures a comprehensive assessment of a model's ability to handle varied levels of difficulty in visual and linguistic reasoning. To capture the rich tapestry of global cultures, ALM-bench carefully curates content from 13 distinct cultural aspects, ranging from traditions and rituals to famous personalities and celebrations. Through this, ALM-bench not only provides a rigorous testing ground for state-of-the-art open and closed-source LMMs but also highlights the importance of cultural and linguistic inclusivity, encouraging the development of models that can serve diverse global populations effectively. Our benchmark is publicly available.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huynh2025b,

title = {CoLLM:ALargeLanguage Model for Composed Image Retrieva},

author = {Chuong Huynh and Jinyu Yang and Ashish Tawari and Mubarak Shah and Son Tran and Raffay Hamid and Trishul Chilimbi and Abhinav Shrivastava},

url = {https://www.amazon.science/publications/collm-a-large-language-model-for-composed-image-retrieval

https://assets.amazon.science/55/e0/7ead8a494dc49fce25cb9947654c/collm-a-large-language-model-for-composed-image-retrieval.pdf

https://collm-cvpr25.github.io/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a complex task that aims to retrieve images based on a multimodal query. Typical training data consists of triplets containing a reference image, a textual description of desired modifications, and the target image, which are expensive and time-consuming to acquire. The scarcity of CIR datasets has led to zero-shot approaches utilizing synthetic triplets or leveraging vision-language models (VLMs) with ubiquitous web-crawled image-caption pairs. However, these methods have significant limitations: synthetic triplets suffer from limited scale, lack of diversity, and unnatural modification text, while image-caption pairs hinder joint embedding learning of the multimodal query due to the absence of triplet data. Moreover, existing approaches struggle with complex and nuanced modification texts that demand sophisticated fusion and understanding of vision and language modalities. We present CoLLM, a one-stop framework that effectively addresses these limitations. Our approach generates triplets on-the-fly from image-caption pairs, enabling supervised training without manual annotation. We leverage Large Language Models (LLMs) to generate joint embeddings of reference images and modification texts, facilitating deeper multimodal fusion. Additionally, we introduce Multi-Text CIR (MTCIR), a large-scale dataset comprising 3.4M samples, and refine existing CIR benchmarks (CIRR and Fashion-IQ) to enhance evaluation reliability. Experimental results demonstrate that CoLLM achieves state-of0the-art performance across multiple CIR benchmarks and settings. MTCIR yields competitive results, with up to 15% performance improvement. Our refined benchmarks provide more reliable evaluation metrics for CIR models, contributing to the advancement of this important field. Project page is at collm-cvpr25.github.io.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025,

title = {M-LLMBasedVideo Frame Selection for Efficient Video Understanding},

author = {Kai Hu and Feng Gao and Xiaohan Nie and Peng Zhou and Son Tran and Tal Neiman and Lingyun Wang and Mubarak Shah and Raffay Hamid and Bing Yin and Trishul Chilimbi},

url = {https://www.amazon.science/publications/m-llm-based-video-frame-selection-for-efficient-video-understanding

https://assets.amazon.science/1a/4e/6794e1dd4fb2acb30eda428a5119/m-llm-based-video-frame-selection-for-efficient-video-understanding.pdf

},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in Multi-Modal Large Language Models (M-LLMs) show promising results in video reasoning. Popular Multi-Modal Large Language Model (M-LLM) frameworks usually apply naive uniform sampling to reduce the number of video frames that are fed into an M-LLM, particularly for long context videos. However, it could lose crucial context in certain periods of a video, so that the downstream M-LLM may not have sufficient visual information to answer a question. To attack this pain point, we propose a light-weight M-LLM-based frame selection method that adaptively select frames that are more relevant to users’ queries. In order to train the proposed frame selector, we introduce two supervision signals (i) Spatial signal, where single frame importance score by prompting a M-LLM; (ii) Temporal signal, in which multiple frames selection by prompting Large Language Model (LLM) using the captions of all frame candidates. The selected frames are then digested by a frozen downstream video M-LLM for visual reasoning and question answering. Empirical results show that the proposed M-LLM video frame selector improves the performances various downstream video Large Language Model (video-LLM) across medium (ActivityNet, NExT-QA) and long (EgoSchema, LongVideoBench) context video question answering benchmarks.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Croitoru2025,

title = {Curriculum Direct Preference Optimization for Diffusion and Consistency Models},

author = {Florinel-Alin Croitoru and Vlad Hondru and Radu Tudor Ionescu and Nicu Sebe and Mubarak Shah},

url = {https://arxiv.org/abs/2405.13637

https://github.com/CroitoruAlin/Curriculum-DPO





},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Direct Preference Optimization (DPO) has been proposed as an effective and efficient alternative to reinforcement learning from human feedback (RLHF). In this paper, we propose a novel and enhanced version of DPO based on curriculum learning for text-to-image generation. Our method is divided into two training stages. First, a ranking of the examples generated for each prompt is obtained by employing a reward model. Then, increasingly difficult pairs of examples are sampled and provided to a text-to-image generative (diffusion or consistency) model. Generated samples that are far apart in the ranking are considered to form easy pairs, while those that are close in the ranking form hard pairs. In other words, we use the rank difference between samples as a measure of difficulty. The sampled pairs are split into batches according to their difficulty levels, which are gradually used to train the generative model. Our approach, Curriculum DPO, is compared against state-of-the-art fine-tuning approaches on nine benchmarks, outperforming the competing methods in terms of text alignment, aesthetics and human preference. Our code is available at this https URL.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025b,

title = {Enhancing Privacy-Utility Trade-offs to Mitigate Memorization in Diffusion Models},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://cvpr.thecvf.com/virtual/2025/poster/34842},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Text-to-image diffusion models have demonstrated remarkable capabilities in creating images highly aligned with user prompts, yet their proclivity for memorizing training set images has sparked concerns about the originality of the generated images and privacy issues, potentially leading to legal complications for both model owners and users, particularly when the memorized images contain proprietary content. Although methods to mitigate these issues have been suggested, enhancing privacy often results in a significant decrease in the utility of the outputs, as indicated by text-alignment scores. To bridge the research gap, we introduce a novel method, PRSS, which refines the classifier-free guidance approach in diffusion models by integrating prompt re-anchoring (PR) to improve privacy and incorporating semantic prompt search (SS) to enhance utility. Extensive experiments across various privacy levels demonstrate that our approach consistently improves the privacy-utility trade-off, establishing a new state-of-the-art.



},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Intent3D: 3D Object Detection in RGB-D Scans Based on Human Intention},

author = {Weitai Kang and Mengxue Qu and Jyoti Kini and Yunchao Wei and Mubarak Shah and Yan Yan},

url = {https://openreview.net/forum?id=5GgjiRzYp3&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/3450_Intent3D_3D_Object_Detect.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In real-life scenarios, humans seek out objects in the 3D world to fulfill their daily needs or intentions. This inspires us to introduce 3D intention grounding, a new task in 3D object detection employing RGB-D, based on human intention, such as “I want something to support my back.” Closely related, 3D visual grounding focuses on understanding human reference. To achieve detection based on human intention, it relies on humans to observe the scene, reason out the target that aligns with their intention (“pillow” in this case), and finally provide a reference to the AI system, such as “A pillow on the couch”. Instead, 3D intention grounding challenges AI agents to automatically observe, reason and detect the desired target solely based on human intention. To tackle this challenge, we introduce the new Intent3D dataset, consisting of 44,990 intention texts associated with 209 f ine-grained classes from 1,042 scenes of the ScanNet [Dai et al., 2017] dataset. We also establish several baselines based on different language-based 3D object detection models on our benchmark. Finally, we propose IntentNet, our unique approach, designed to tackle this intention-based detection problem. It focuses on three key aspects: intention understanding, reasoning to identify object candidates, and cascaded adaptive learning that leverages the intrinsic priority logic of different losses for multiple objective optimization.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chhipa2025,

title = {ASTrA: Adversarial Self-supervised Training with Adaptive-Attacks},

author = {Prakash Chandra Chhipa and Gautam Vashishtha and Settur Jithamanyu and Rajkumar Saini and Mubarak Shah and Marcus Liwicki},

url = {https://prakashchhipa.github.io/projects/ASTrA

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7338_ASTrA_Adversarial_Self_su.pdf.pdf},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Existing self-supervised adversarial training (self-AT) methods rely on handcrafted adversarial attack strategies for PGD attacks, which fail to adapt to the evolving learning dynamics of the model and do not account for instancespecific characteristics of images. This results in sub-optimal adversarial robustness and limits the alignment between clean and adversarial data distributions. To address this, we propose ASTrA (Adversarial Self-supervised Training with Adaptive-Attacks), a novel framework introducing a learnable, self-supervised attack strategy network that autonomously discovers optimal attack parameters through exploration-exploitation in a single training episode. ASTrA leverages a reward mechanism based on contrastive loss, optimized with REINFORCE, enabling adaptive attack strategies without labeled data or additional hyperparameters. We further introduce a mixed contrastive objective to align the distribution of clean and adversarial examples in representation space. ASTrA achieves state-of-the-art results on CIFAR10, CIFAR100, and STL10 while integrating seamlessly as a plug-and-play module for other self-AT methods. ASTrAshows scalability to larger datasets, demonstrates strong semi-supervised performance, and is resilient to robust overfitting, backed by explainability analysis on optimal attack strategies. Project page for source code and other details at https://prakashchhipa.github.io/projects/ASTrA.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2025,

title = {ALBAR: Adversarial Learning approach to mitigate Biases in Action Recognition},

author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://openreview.net/forum?id=9KiE3t6CsL&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7609_ALBAR_Adversarial_Learnin.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Bias in machine learning models can lead to unfair decision making, and while it has been well-studied in the image and text domains, it remains underexplored in action recognition. Action recognition models often suffer from background bias (i.e., inferring actions based on background cues) and foreground bias (i.e., relying on subject appearance), which can be detrimental to real-life applications such as autonomous vehicles or assisted living monitoring. While prior approaches have mainly focused on mitigating background bias using specialized augmentations, we thoroughly study both biases. We propose approachname, a novel adversarial training method that mitigates foreground and background biases without requiring specialized knowledge of the bias attributes. Our framework applies an adversarial cross-entropy loss to the sampled static clip (where all the frames are the same) and aims to make its class probabilities uniform using a proposed textit{entropy maximization} loss. Additionally, we introduce a textit{gradient penalty} loss for regularization against the debiasing process. We evaluate our method on established background and foreground bias protocols, setting a new state-of-the-art and strongly improving combined debiasing performance by over textbf{12%} on HMDB51. Furthermore, we identify an issue of background leakage in the existing UCF101 protocol for bias evaluation which provides a shortcut to predict actions and does not provide an accurate measure of the debiasing capability of a model. We address this issue by proposing more fine-grained segmentation boundaries for the actor, where our method also outperforms existing approaches.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2025,

title = {AdaIR: Adaptive All-in-One Image Restoration via Frequency Mining and Modulation},

author = {Yuning Cui and Syed Waqas Zamir and Salman Khan and Alois Knoll and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://openreview.net/forum?id=M5t0WvjfCg&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/8311_AdaIR_Adaptive_All_in_One.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In the image acquisition process, various forms of degradation, including noise, blur, haze, and rain, are frequently introduced. These degradations typically arise from the inherent limitations of cameras or unfavorable ambient conditions. To recover clean images from their degraded versions, numerous specialized restoration methods have been developed, each targeting a specific type of degradation. Recently, all-in-one algorithms have garnered significant attention by addressing different types of degradations within a single model without requiring the prior information of the input degradation type. However, most methods purely operate in the spatial domain and do not delve into the distinct frequency variations inherent to different degradation types. To address this gap, we propose an adaptive all-in-one image restoration network based on frequency mining and modulation. Our approach is motivated by the observation that different degradation types impact the image content on different frequency subbands, thereby requiring different treatments for each restoration task. Specifically, we first mine low- and high-frequency information from the input features, guided by the adaptively decoupled spectra of the degraded image. The extracted features are then modulated by a bidirectional operator to facilitate interactions between different frequency components. Finally, the modulated features are merged into the original input for a progressively guided restoration. With this approach, the model achieves adaptive reconstruction by accentuating the informative frequency subbands according to different input degradations. Extensive experiments demonstrate that the proposed method, named AdaIR, achieves state-of-the-art performance on different image restoration tasks, including image denoising, dehazing, deraining, motion deblurring, and low-light image enhancement. Our code and models will be made publicly available.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025,

title = {Exploring Local Memorization in Diffusion Models via Bright Ending Attention},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://openreview.net/forum?id=p4cLtzk4oe&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/9283_Exploring_Local_Memorizat.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In this paper, we identify and leverage a novel `bright ending' (BE) anomaly in diffusion models prone to memorizing training images to address a new task: locating localized memorization regions within these models. BE refers to a distinct cross-attention pattern observed in text-to-image generations using diffusion models. Specifically, memorized image patches exhibit significantly greater attention to the end token during the final inference step compared to non-memorized patches. This attention map effectively highlights regions where the generated image replicates training data. Furthermore, driven by our observation that local memorization significantly underperforms in existing tasks of measuring, detecting, and mitigating memorization in diffusion models compared to global memorization, we propose a simple yet effective method to integrate BE and the results of the new localization task into these existing frameworks. This integration effectively improves their performances by narrowing the performance gap caused by local memorization. Our results not only demonstrate the successful execution of the new localization task but also establish new state-of-the-art performance across all existing tasks, underscoring the significance of the BE phenomenon.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wu2024,

title = {PTQ4DiT: Post-training Quantization for Diffusion Transformers},

author = {Junyi Wu and Haoxuan Wang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://nips.cc/virtual/2024/poster/95445

https://arxiv.org/pdf/2405.16005

https://github.com/adreamwu/PTQ4DiT},

year  = {2024},

date = {2024-12-13},

urldate = {2024-12-13},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The recent introduction of Diffusion Transformers (DiTs) has demonstrated exceptional capabilities in image generation by using a different backbone architecture, departing from traditional U-Nets and embracing the scalable nature of transformers. Despite their advanced capabilities, the wide deployment of DiTs, particularly for real-time applications, is currently hampered by considerable computational demands at the inference stage. Post-training Quantization (PTQ) has emerged as a fast and data-efficient solution that can significantly reduce computation and memory footprint by using low-bit weights and activations. However, its applicability to DiTs has not yet been explored and faces non-trivial difficulties due to the unique design of DiTs. In this paper, we propose PTQ4DiT, a specifically designed PTQ method for DiTs. We discover two primary quantization challenges inherent in DiTs, notably the presence of salient channels with extreme magnitudes and the temporal variability in distributions of salient activation over multiple timesteps. To tackle these challenges, we propose Channel-wise Salience Balancing (CSB) and Spearmen's 

ρ

-guided Salience Calibration (SSC). CSB leverages the complementarity property of channel magnitudes to redistribute the extremes, alleviating quantization errors for both activations and weights. SSC extends this approach by dynamically adjusting the balanced salience to capture the temporal variations in activation. Additionally, to eliminate extra computational costs caused by PTQ4DiT during inference, we design an offline re-parameterization strategy for DiTs. Experiments demonstrate that our PTQ4DiT successfully quantizes DiTs to 8-bit precision (W8A8) while preserving comparable generation ability and further enables effective quantization to 4-bit weight precision (W4A8) for the first time.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2024,

title = {Sparse Points to Dense Clouds: Enhancing 3D Detection with Limited LiDAR Data},

author = {Aakash Kumar and Chen Chen and Ajmal Mian and Neils Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2404.06715v1.pdf

https://aakashjuseja-aj.github.io/Sparse_to_Dense/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

abstract = {3D detection is a critical task that enables machines to identify and locate objects in three-dimensional space. It has a broad range of applications in several fields, including autonomous driving, robotics and augmented reality. Monocular 3D detection is attractive as it requires only a single camera, however, it lacks the accuracy and robustness required for real world applications. High resolution LiDAR on the other hand, can be expensive and lead to interference problems in heavy traffic given their active transmissions. We propose a balanced approach that combines the advantages of monocular and point cloud-based 3D detection. Our method requires only a small number of 3D points, that can be obtained from a low-cost, low-resolution sensor. Specifically, we use only 512 points, which is just 1% of a full LiDAR frame in the KITTI dataset. Our method reconstructs a complete 3D point cloud from this limited 3D information combined with a single image. The reconstructed 3D point cloud and corresponding image can be used by any multi-modal off-the-shelf detector for 3D object detection. By using the proposed network architecture with an off-the-shelf multi-modal 3D detector, the accuracy of 3D detection improves by 20% compared to the state-of-the-art monocular detection methods and 6% to 9% compare to the baseline multi-modal methods on KITTI and JackRabbot datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar

Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf

https://prakashchhipa.github.io/projects/mpd

https://youtu.be/MKh9NE_XEMY

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_presentation_ECCV2024_final.pdf},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method  outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200

},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf

https://daveishan.github.io/finepsuedo-webpage/

https://youtu.be/bWOd8_JpjQs?si=WWRDhdg5ADWL0uwB},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias,  classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48,  FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the  robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf

https://arxiv.org/abs/2407.13851

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and benjamin Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024b,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf

https://daveishan.github.io/avr-webpage/

https://youtu.be/6euQwz7XdQk?si=v12dQH4e7UrTUrIU},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf

https://arxiv.org/pdf/2407.04370

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf

https://arxiv.org/abs/2408.02840

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf

https://parthpk.github.io/cityguessr-webpage/

},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38]. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2024,

title = {VidLA: Video-Language Alignment at Scale},

author = {Mamshad Nayeem Rizve and Fan Fei and Jayakrishnan Unnikrishnan and Son Tran and Benjamin Z. Yao and Belinda Zeng and Mubarak Shah and Trishul Chilimbi},

url = {https://arxiv.org/abs/2403.14870},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we propose VidLA, an approach for video-language alignment at scale. There are two major limitations of previous video-language alignment approaches. First, they do not capture both short-range and long-range temporal dependencies and typically employ complex hierarchical deep network architectures that are hard to integrate with existing pretrained image-text foundation models. To effectively address this limitation, we instead keep the network architecture simple and use a set of data tokens that operate at different temporal resolutions in a hierarchical manner, accounting for the temporally hierarchical nature of videos. By employing a simple two-tower architecture, we are able to initialize our video-language model with pretrained image-text foundation models, thereby boosting the final performance. Second, existing video-language alignment works struggle due to the lack of semantically aligned large-scale training data. To overcome it, we leverage recent LLMs to curate the largest video-language dataset to date with better visual grounding. Furthermore, unlike existing video-text datasets which only contain short clips, our dataset is enriched with video clips of varying durations to aid our temporally hierarchical data tokens in extracting better representations at varying temporal scales. Overall, empirical results show that our proposed approach surpasses state-of-the-art methods on multiple retrieval benchmarks, especially on longer videos, and performs competitively on classification benchmarks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ristea2024,

title = {Self-Distilled Masked Auto-Encoders are Efficient Video Anomaly Detectors},

author = {Nicolae Catalin Ristea and Florinel Alin Croitoru and Radu Tudor Ionescu and Marius Popescu and Fahad Khan and Mubarak Shah},

url = {https://arxiv.org/abs/2306.12041

https://github.com/ristea/aed-mae},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {We propose an efficient abnormal event detection model based on a lightweight masked auto-encoder (AE) applied at the video frame level. The novelty of the proposed model is threefold. First, we introduce an approach to weight tokens based on motion gradients, thus shifting the focus from the static background scene to the foreground objects. Second, we integrate a teacher decoder and a student decoder into our architecture, leveraging the discrepancy between the outputs given by the two decoders to improve anomaly detection. Third, we generate synthetic abnormal events to augment the training videos, and task the masked AE model to jointly reconstruct the original frames (without anomalies) and the corresponding pixel-level anomaly maps. Our design leads to an efficient and effective model, as demonstrated by the extensive experiments carried out on four benchmarks: Avenue, ShanghaiTech, UBnormal and UCSD Ped2. The empirical results show that our model achieves an excellent trade-off between speed and accuracy, obtaining competitive AUC scores, while processing 1655 FPS. Hence, our model is between 8 and 70 times faster than competing methods. We also conduct an ablation study to justify our design. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2024,

title = {Composed Video Retrieval via Enriched Context and Discriminative Embeddings},

author = {Omkar Chakradhar Thawakar and Muzammal Naseer and Rao Muhammad Anwer and Salman Khan and Michael Felsberg and Mubarak Shah and Fahad Khan },

url = {https://arxiv.org/abs/2403.16997

https://github.com/OmkarThawakar/composed-video-retrieval},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Composed video retrieval (CoVR) is a challenging problem in computer vision which has recently highlighted the integration of modification text with visual queries for more sophisticated video search in large databases. Existing works predominantly rely on visual queries combined with modification text to distinguish relevant videos. However, such a strategy struggles to fully preserve the rich query-specific context in retrieved target videos and only represents the target video using visual embedding. We introduce a novel CoVR framework that leverages detailed language descriptions to explicitly encode query-specific contextual information and learns discriminative embeddings of vision only, text only and vision-text for better alignment to accurately retrieve matched target videos. Our proposed framework can be flexibly employed for both composed video (CoVR) and image (CoIR) retrieval tasks. Experiments on three datasets show that our approach obtains state-of-the-art performance for both CovR and zero-shot CoIR tasks, achieving gains as high as around 7% in terms of recall@K=1 score.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dutta2024,

title = {Multiview Aerial Visual RECognition (MAVREC) Dataset: Can Multi-view Improve Aerial Visual Perception?},

author = {Aritra Dutta and Srijan Das and Jacob Nielsen and Rajatsubhra Chakraborty and Mubarak Shah},

url = {https://arxiv.org/abs/2312.04548

https://mavrec.github.io/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite the commercial abundance of UAVs, aerial data acquisition remains challenging, and the existing Asia and North America-centric open-source UAV datasets are small-scale or low-resolution and lack diversity in scene contextuality. Additionally, the color content of the scenes, solar-zenith angle, and population density of different geographies influence the data diversity. These two factors conjointly render suboptimal aerial-visual perception of the deep neural network (DNN) models trained primarily on the ground-view data, including the open-world foundational models.



To pave the way for a transformative era of aerial detection, we present Multiview Aerial Visual RECognition or MAVREC, a video dataset where we record synchronized scenes from different perspectives -- ground camera and drone-mounted camera. MAVREC consists of around 2.5 hours of industry-standard 2.7K resolution video sequences, more than 0.5 million frames, and 1.1 million annotated bounding boxes. This makes MAVREC the largest ground and aerial-view dataset, and the fourth largest among all drone-based datasets across all modalities and tasks. Through our extensive benchmarking on MAVREC, we recognize that augmenting object detectors with ground-view images from the corresponding geographical location is a superior pre-training strategy for aerial detection. Building on this strategy, we benchmark MAVREC with a curriculum-based semi-supervised object detection approach that leverages labeled (ground and aerial) and unlabeled (only aerial) images to enhance the aerial detection.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024c,

title = {CodaMal: Contrastive Domain Adaptation for Malaria Detection in Low-Cost Microscopes },

author = {Ishan Rajendrakumar Dave and Tristan de Blegiers and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2402.10478

https://daveishan.github.io/codamal-webpage/},

year  = {2024},

date = {2024-02-16},

publisher = {31st IEEE International Conference on Image Processing (ICIP) (Oral)},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {No More Shortcuts: Realizing the Potential of Temporal Self-Supervision},

author = {Ishan Rajendrakumar Dave and Simon Jenni and Mubarak Shah},

url = {https://arxiv.org/pdf/2312.13008

https://daveishan.github.io/nms-webpage/

https://youtu.be/5MBnxmMBQh0},

year  = {2023},

date = {2023-12-20},

publisher = {AAAI Conference on Artificial Intelligence (AAAI), Main Technical Track},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cepeda2023,

title = {GeoCLIP: Clip-Inspired Alignment between Locations and Images for Effective Worldwide Geo-localization},

author = {Vicente Vivanco Cepeda and Gaurav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_paper.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_supplementary.pdf

https://vicentevivan.github.io/GeoCLIP/},

year  = {2023},

date = {2023-12-11},

publisher = {Thirty-seventh Conference on Neural Information Processing Systems},

abstract = {Worldwide Geo-localization aims to pinpoint the precise location of images taken anywhere on Earth. This task has considerable challenges due to immense variation in geographic landscapes. The image-to-image retrieval-based approaches fail to solve this problem on a global scale as it is not feasible to construct a large gallery of images covering the entire world. Instead, existing approaches divide the globe into discrete geographic cells, transforming the problem into a classification task. However, their performance is limited by the predefined classes and often results in inaccurate localizations when an image’s location significantly deviates from its class center. To overcome these limitations, we propose GeoCLIP, a novel CLIP-inspired Image-to-GPS retrieval approach that enforces alignment between the image and its corresponding GPS locations. GeoCLIP’s location encoder models the Earth as a continuous function by employing positional encoding through random Fourier features and constructing a hierarchical representation that captures information at varying resolutions to yield a semantically rich highdimensional feature suitable to use even beyond geo-localization. To the best of our knowledge, this is the first work employing GPS encoding for geo-localization. We demonstrate the efficacy of our method via extensive experiments and ablations on benchmark datasets. We achieve competitive performance with just 20% of training data, highlighting its effectiveness even in limited-data settings. Furthermore, we qualitatively demonstrate geo-localization using a text query by leveraging CLIP backbone of our image encoder.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Frequency Domain Adversarial Training for Robust Volumetric Medical Segmentation},

author = {Asif Hanif and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Frequency-Domain-Adversarial-Training-for-Robust-Volumetric-Medical-Segmentation.pdf

https://github.com/asif-hanif/vafa},

doi = {https://doi.org/10.48550/arXiv.2307.07269},

year  = {2023},

date = {2023-10-08},

publisher = {The 26th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2023},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023b,

title = {When Do Curricula Work in Federated Learning? },

author = {Saeed Vahidian and Sreevatsank Kadaveru and Woonjoon Baek and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.12712.pdf

https://arxiv.org/abs/2212.12712},

doi = {https://doi.org/10.48550/arXiv.2212.12712},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {An oft-cited open problem of federated learning is the existence of data heterogeneity at the clients. One pathway to understanding the drastic accuracy drop in federated learning is by scrutinizing the behavior of the clients' deep models on data with different levels of "difficulty", which has been left unaddressed. In this paper, we investigate a different and rarely studied dimension of FL: ordered learning. Specifically, we aim to investigate how ordered learning principles can contribute to alleviating the heterogeneity effects in FL. We present theoretical analysis and conduct extensive empirical studies on the efficacy of orderings spanning three kinds of learning: curriculum, anti-curriculum, and random curriculum. We find that curriculum learning largely alleviates non-IIDness. Interestingly, the more disparate the data distributions across clients the more they benefit from ordered learning. We provide analysis explaining this phenomenon, specifically indicating how curriculum training appears to make the objective landscape progressively less convex, suggesting fast converging iterations at the beginning of the training procedure. We derive quantitative results of convergence for both convex and nonconvex objectives by modeling the curriculum training on federated devices as local SGD with locally biased stochastic gradients. Also, inspired by ordered learning, we propose a novel client selection technique that benefits from the real-world disparity in the clients. Our proposed approach to client selection has a synergic effect when applied together with ordered learning in FL.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Samarasinghe2023,

title = {CDFSL-V: Cross-Domain Few-Shot Learning for Videos},

author = {Sarinda Samarasinghe and Mamshad Nayeem and Rizve Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CDFSL_Video_Combined_Final.pdf

https://sarinda251.github.io/CDFSL-V-site/

https://www.youtube.com/watch?v=RdlEzfW013o},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Few-shot video action recognition is an effective approach to recognizing new categories with only a few labeled examples, thereby reducing the challenges associated with collecting and annotating large-scale video datasets. Existing methods in video action recognition rely on large labeled datasets from the same domain. However, this setup is not realistic as novel categories may come from different data domains that may have different spatial and temporal characteristics. This dissimilarity between the source and target domains can pose a significant challenge, rendering traditional few-shot action recognition techniques ineffective. To address this issue, in this work, we propose a novel cross-domain few-shot video action recognition method that leverages self-supervised learning and curriculum learning to balance the information from the source and target domains. To be particular, our method employs a masked autoencoder-based self-supervised training objective to learn from both source and target data in a self-supervised manner. Then a progressive curriculum balances learning the discriminative information from the source dataset with the generic information learned from the target domain. Initially, our curriculum utilizes supervised learning to learn class discriminative features from the source data. As the training progresses, we transition to learning target-domain-specific features. We propose a progressive curriculum to encourage the emergence of rich features in the target domain based on class discriminative supervised features in the source domain. %a schedule that helps with this transition. We evaluate our method on several challenging benchmark datasets and demonstrate that our approach outperforms existing cross-domain few-shot learning techniques.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Preserving Modality Structure Improves Multi-Modal Learning },

author = {Swetha Sirnam and Mamshad Nayeem Rizve and Hilde Kuhne and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.13077.pdf

https://arxiv.org/abs/2308.13077

https://github.com/Swetha5/Multi_Sinkhorn_Knopp

https://swetha5.github.io/MultiSK/

https://youtu.be/1CrGkUATy50

},

doi = {https://doi.org/10.48550/arXiv.2308.13077},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Self-supervised learning on large-scale multi-modal datasets allows learning semantically meaningful embeddings in a joint multi-modal representation space without relying on human annotations. These joint embeddings enable zero-shot cross-modal tasks like retrieval and classification. However, these methods often struggle to generalize well on out-of-domain data as they ignore the semantic structure present in modality-specific embeddings. In this context, we propose a novel Semantic-Structure-Preserving Consistency approach to improve generalizability by preserving the modality-specific relationships in the joint embedding space. To capture modality-specific semantic relationships between samples, we propose to learn multiple anchors and represent the multifaceted relationship between samples with respect to their relationship with these anchors. To assign multiple anchors to each sample, we propose a novel Multi-Assignment Sinkhorn-Knopp algorithm. Our experimentation demonstrates that our proposed approach learns semantically meaningful anchors in a self-supervised manner. Furthermore, our evaluation on MSR-VTT and YouCook2 datasets demonstrates that our proposed multi-anchor assignment based solution achieves state-of-the-art performance and generalizes to both inand out-of-domain datasets. Code: https://github.com/Swetha5/Multi_Sinkhorn_Knopp},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023b,

title = {Diffusion Action Segmentation},

author = {Daochang Liu and Qiyue Li and Anh-Dung Dinh and Tingting Jiang and Mubarak Shah and Chang Xu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.17959.pdf

https://arxiv.org/abs/2303.17959

https://finspire13.github.io/DiffAct-Project-Page/

https://github.com/Finspire13/DiffAct

https://youtu.be/o_Jp8shth7U

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Slides.pptx},

doi = { https://doi.org/10.48550/arXiv.2303.17959},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Temporal action segmentation is crucial for understanding long-form videos. Previous works on this task commonly adopt an iterative refinement paradigm by using multi-stage models. We propose a novel framework via denoising diffusion models, which nonetheless shares the same inherent spirit of such iterative refinement. In this framework, action predictions are iteratively generated from random noise with input video features as conditions. To enhance the modeling of three striking characteristics of human actions, including the position prior, the boundary ambiguity, and the relational dependency, we devise a unified masking strategy for the conditioning inputs in our framework. Extensive experiments on three benchmark datasets, i.e., GTEA, 50Salads, and Breakfast, are performed and the proposed method achieves superior or comparable results to state-of-the-art methods, showing the effectiveness of a generative approach for action segmentation.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2023,

title = {TeD-SPAD: Temporal Distinctiveness for Self-supervised Privacy-preservation for video Anomaly Detection},

author = {Joseph Fioresi and Ishan Dave and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.11072.pdf

https://arxiv.org/abs/2308.11072

https://github.com/UCF-CRCV/TeD-SPAD

https://joefioresi718.github.io/TeD-SPAD_webpage/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TeDSPAD_ICCV_poster.pdf

https://youtu.be/3a9qeJUD1GU},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Video-FocalNets: Spatio-Temporal Focal Modulation for Video Action Recognition },

author = {Syed Talal Wasim and Muhammad Uzair Khattak and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.06947.pdf

https://arxiv.org/abs/2307.06947

https://talalwasim.github.io/Video-FocalNets/

https://github.com/TalalWasim/Video-FocalNets

https://talalwasim.github.io/Video-FocalNets/#BibTeX},

doi = { https://doi.org/10.48550/arXiv.2307.06947},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Recent video recognition models utilize Transformer models for long-range spatio-temporal context modeling. Video transformer designs are based on self-attention that can model global context at a high computational cost. In comparison, convolutional designs for videos offer an efficient alternative but lack long-range dependency modeling. Towards achieving the best of both designs, this work proposes Video-FocalNet, an effective and efficient architecture for video recognition that models both local and global contexts. Video-FocalNet is based on a spatio-temporal focal modulation architecture that reverses the interaction and aggregation steps of self-attention for better efficiency. Further, the aggregation step and the interaction step are both implemented using efficient convolution and element-wise multiplication operations that are computationally less expensive than their self-attention counterparts on video representations. We extensively explore the design space of focal modulation-based spatio-temporal context modeling and demonstrate our parallel spatial and temporal encoding design to be the optimal choice. Video-FocalNets perform favorably well against the state-of-the-art transformer-based models for video recognition on three large-scale datasets (Kinetics-400, Kinetics-600, and SS-v2) at a lower computational cost. Our code/models are publicly released.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Mitochondria Instance Segmentation with Spatio-Temporal Transformers},

author = {Omkar Thawakar and Rao Muhammad Anwer and Jorma Laaksonen and Orly Reiner and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.12073.pdf

https://github.com/OmkarThawakar/STT-UNET

https://arxiv.org/pdf/2303.12073.pdf

https://link.springer.com/chapter/10.1007/978-3-031-43993-3_59

},

doi = {https://doi.org/10.1007/978-3-031-43993-3_59},

isbn = {978-3-031-43993-3},

year  = {2023},

date = {2023-10-01},

booktitle = {Lecture Notes in Computer Science},

journal = {arXiv:2303.12073},

volume = {14227},

publisher = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},

abstract = {Accurate 3D mitochondria instance segmentation in electron microscopy (EM) is a challenging problem and serves as a prerequisite to empirically analyze their distributions and morphology. Most existing approaches employ 3D convolutions to obtain representative features. However, these convolution-based approaches struggle to effectively capture long-range dependencies in the volume mitochondria data, due to their limited local receptive field. To address this, we propose a hybrid encoder-decoder framework based on a split spatio-temporal attention module that efficiently computes spatial and temporal self-attentions in parallel, which are later fused through a deformable convolution. Further, we introduce a semantic foreground-background adversarial loss during training that aids in delineating the region of mitochondria instances from the background clutter. Our extensive experiments on three benchmarks, Lucchi, MitoEM-R and MitoEM-H, reveal the benefits of the proposed contributions achieving state-of-the-art results on all three datasets. Our code and models are available at https://github.com/OmkarThawakar/STT-UNET.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{deBlegiers2023,

title = {EventTransAct: A Video Transformer-based Framework for Event-camera Based Action Recognition},

author = {Tristan de Blegiers and Ishan Rajendrakumar Dave and Adeel Yousaf and Mubarak Shah},

url = {https://arxiv.org/pdf/2308.13711

https://github.com/tristandb8/EventTransAct

https://tristandb8.github.io/EventTransAct_webpage/

https://www.youtube.com/watch?v=YCff-rTrgco},

year  = {2023},

date = {2023-08-25},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023,

title = {R2Former: Unified retrieval and ranking Transformer for Place Recognition},

author = {Sijie Zhu and Linjie Yang and Chen Chen and Mubarak Shah and Xiaohui Shen and Heng Wang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR_2023_PlaceRecognitionFinal.pdf

https://arxiv.org/pdf/2304.03410.pdf

https://github.com/Jeff-Zilence/R2Former},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Visual Place Recognition (VPR) estimates the location of query images by matching them with images in a reference database. Conventional methods generally adopt aggregated CNN features for global retrieval and RANSAC-based geometric verification for reranking. However, RANSAC only considers geometric information but ignores other possible information that could be useful for reranking, e.g. local feature correlation, and attention values. In this paper, we propose a unified place recognition framework that handles both retrieval and reranking with a novel transformer model, named R2Former. The proposed reranking module takes feature correlation, attention value, and xy coordinates into account, and learns to determine whether the image pair is from the same location. The whole pipeline is end-to-end trainable and the reranking module alone can

also be adopted on other CNN or transformer backbones as a generic component. Remarkably, R2Former significantly

outperforms state-of-the-art methods on major VPR datasets with much less inference time and memory consumption.

It also achieves the state-of-the-art on the holdout MSLS challenge set and could serve as a simple yet strong solution for real-world large-scale applications. Experiments also show vision transformer tokens are comparable and sometimes better than CNN local features on local matching. The code will be publicly available. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023b,

title = {Class Prototypes based Contrastive Learning for Classifying Multi-Label and Fine-Grained Educational Videos},

author = {Rohit Gupta and Anirban Roy and Sujeong Kim and Claire Christensen and Todd Grindal and Sarah Nixon Gerard and Madeline Cincebeaux and Ajay Divakaran and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Rohit_SRI_CVPR2023_Multi_Modal_Multi_Label_Contrastive_Learning_Camera_Ready-4.pdf

https://www.rohitg.xyz/MMContrast/

https://nusci.csl.sri.com/project/APPROVE},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The recent growth in the consumption of online media by children during early childhood necessitates data-driven tools enabling educators to filter out appropriate educational content for young learners. This paper presents an approach for detecting educational content in online videos. We focus on two widely used educational content classes: literacy and math. For each class, we choose prominent codes (sub-classes) based on the Common Core Standards. For example, literacy codes include ‘letter names’, ‘letter sounds’, and math codes include ‘counting’, ‘sorting’. We pose this as a fine-grained multilabel classification problem as videos can contain multiple types of educational content and the content classes can get visually similar (e.g., ‘letter names’ vs ‘letter sounds’). We propose a novel class prototypes based supervised contrastive learning approach that can handle fine-grained samples associated with multiple labels. We learn a class prototype for each class and a loss function is employed to minimize the distances between a class prototype and the samples from the class. Similarly,

distances between a class prototype and the samples from other classes are maximized. As the alignment between visual

and audio cues are crucial for effective comprehension, we consider a multimodal transformer network to capture the interaction between visual and audio cues in videos while learning the embedding for videos. For evaluation, we present a dataset, APPROVE, employing educational videos from YouTube labeled with fine-grained education classes by education researchers. APPROVE consists of 193 hours of expert-annotated videos with 19 classes. The proposed approach outperforms strong baselines on APPROVE and other benchmarks such as Youtube-8M, and COIN. The dataset is available at https://nusci.csl.sri.com/project/APPROVE.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2023,

title = {TimeBalance: Temporally-Invariant and Temporally-Distinctive Video Representations for Semi-Supervised Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TimeBalance_CVPR23_arxiv.pdf

https://daveishan.github.io/timebalance_webpage/

https://github.com/DAVEISHAN/TimeBalance},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Semi-Supervised Learning can be more beneficial for the video domain compared to images because of its higher annotation

cost and dimensionality. Besides, any video understanding task requires reasoning over both spatial and temporal dimensions. In order to learn both the static and motion related features for the semi-supervised action recognition task, existing methods rely on hard input inductive biases like using two-modalities (RGB and Optical-flow) or two-stream of different playback rates.

Instead of utilizing unlabeled videos through diverse input streams, we rely on self-supervised video representations,

particularly, we utilize temporally-invariant and temporally-distinctive representations. We observe that these representations complement each other depending on the nature of the action. Based on this observation, we propose a student-teacher semi-supervised learning framework, TimeBalance, where we distill the knowledge from a temporally-invariant and a temporally-distinctive teacher. Depending on the nature of the unlabeled video, we dynamically combine the knowledge of these two  teachers based on a novel temporal similarity-based reweighting scheme. Our method achieves state-of-the-art performance

on three action recognition benchmarks: UCF101, HMDB51, and Kinetics400. Code: https://github.com/DAVEISHAN/TimeBalance.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2023,

title = {PivoTAL: Prior-Driven Supervision for Weakly-Supervised Temporal Action Localization},

author = {Mamshad Nayeem Rizve and Gaurav Mittal and Ye Yu and Matthew Hall and Sandra Sajeev and Mubarak Shah and Mei Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023_Supplemental_Material.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR2023_Poster.pdf

https://www.youtube.com/watch?v=6kAoQjXfzio},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Urooj2023,

title = {Learning Situation Hyper-Graphs for Video Question Answering},

author = {Aisha Urooj and Hilde Kuehne and Bo Wu and Kim Chheu and Walid Bousselham and Chuang Gan and Niels Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2023072364-4.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SHG_VQA_CVPR2023_cam_ready_supp.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Answering questions about complex situations in videos requires not only capturing the presence of actors, objects, and their relations but also the evolution of these relationships over time. A situation hyper-graph is a representation that describes situations as scene sub-graphs for video frames and hyper-edges for connected sub-graphs and has been proposed to capture all such information in a compact structured form. In this work, we propose an architecture for Video Question Answering (VQA) that enables answering questions related to video content by predicting situation hyper-graphs, coined Situation Hyper-Graph based Video Question Answering (SHG-VQA). To this end, we train a situation hyper-graph decoder to implicitly identify graph representations with actions and object/human-object relationships from the input video clip. and to use cross-attention

between the predicted situation hyper-graphs and the question embedding to predict the correct answer. The proposed

method is trained in an end-to-end manner and optimized by a VQA loss with the cross-entropy function and a Hungarian

matching loss for the situation graph prediction. The effectiveness of the proposed architecture is extensively evaluated

on two challenging benchmarks: AGQA and STAR. Our results show that learning the underlying situation hypergraphs

helps the system to significantly improve its performance for novel challenges of video question-answering tasks. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bhunia2023,

title = {Person Image Synthesis via Denoising Diffusion Model},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Jorma Tapio Laaksonen and Mubarak Shah and Fahad Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/person_image_synthesis_via_den-Camera-ready-PDF.pdf

https://lnkd.in/d-8v3r8B

https://lnkd.in/dGPTjvge

https://lnkd.in/dxcGQsUX

https://github.com/ankanbhunia/PIDM},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The pose-guided person image generation task requires synthesizing photorealistic images of humans in arbitrary poses. The existing approaches use generative adversarial networks that do not necessarily maintain realistic textures or need dense correspondences that struggle to handle complex  deformations and severe occlusions. In this work, we show how denoising diffusion models can be applied for high-fidelity person image synthesis with

strong sample diversity and enhanced mode coverage of the learnt data distribution. Our proposed Person Image Diffusion Model (PIDM) disintegrates the complex transfer problem into a series of simpler forward-backward denoising steps. This helps in learning plausible source-to-target transformation trajectories that result in faithful textures and undistorted appearance details. We introduce a ‘texture diffusion module’ based on cross-attention to accurately model the correspondences between appearance and pose information available in source and target images. Further, we propose  disentangled classifier-free guidance’ to ensure close resemblance between the conditional inputs and the synthesized output in terms of both pose and appearance information. Our extensive results on two large-scale benchmarks and a user study demonstrate the photorealism of our proposed approach under challenging scenarios. We also show how our generated images can help in downstream tasks. Code is available at https://github.com/ankanbhunia/PIDM.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wasim2023,

title = {Vita-CLIP: Video and text adaptive CLIP via Multimodal Prompting},

author = {Syed Talal Wasim and Muzammal Naseer and Salman Khan and Fahad Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/vita_clip_video_and_text_adapt-Camera-ready-PDF.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Adopting contrastive image-text pretrained models like CLIP towards video classification has gained attention due to its cost-effectiveness and competitive performance. However, recent works in this area face a trade-off. Finetuning the pretrained model to achieve strong supervised performance results in low zero-shot generalization. Similarly, freezing the backbone to retain zero-shot capability causes significant drop in supervised accuracy. Because of this,

recent works in literature typically train separate models for supervised and zero-shot action recognition. In this work, we propose a multimodal prompt learning scheme that works to balance the supervised and zero-shot performance under a single unified training. Our prompting approach on the vision side caters for three aspects: 1) Global video-level prompts to model the data distribution; 2) Local frame-level prompts to provide per-frame discriminative

conditioning; and 3) a summary prompt to extract a condensed video representation. Additionally, we define a prompting scheme on the text side to augment the textual context. Through this prompting scheme, we can achieve state-of-the-art zero-shot performance on Kinetics-600, HMDB51 and UCF101 while remaining competitive in the supervised setting. By keeping the pretrained backbone frozen, we optimize a much lower number of parameters and retain the existing general representation which helps achieve the strong zero-shot performance. Our codes and models will be publicly released. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Clark2023,

title = {Where We Are and What We're Looking At: Query Based Worldwide Image Geo-localization Using Hierarchies and Scenes},

author = {Brandon Eric Clark and Alec Kerrigan and Parth Parag Kulkarni and Vicente Vivanco Cepeda and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Camera-Ready-Full-Paper.pdf

https://github.com/AHKerrigan/GeoGuessNet

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR23-Poster_THU-PM-246-1.pdf

https://www.youtube.com/watch?v=fp3hZGbwPqk},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Determining the exact latitude and longitude that a photo was taken is a useful and widely applicable task, yet it remains exceptionally difficult despite the accelerated progress of other computer vision tasks. Most previous approaches have opted to learn single representations of query images, which are then classified at different levels of geographic granularity. These approaches fail to exploit the different visual cues that give context to different hierarchies, such as the country, state, and city level. To this end, we introduce an end-to-end transformer-based architecture that exploits the relationship between different geographic levels (which we refer to as hierarchies) and the corresponding visual scene information in an image through hierarchical cross-attention. We achieve this by learning a query for each geographic hierarchy and scene type. Furthermore, we learn a separate representation for different environmental scenes, as different scenes in the same location are often defined by completely different visual features. We achieve state of the art accuracy on 4 standard geo-localization datasets : Im2GPS, Im2GPS3k, YFCC4k, and YFCC26k, as well as qualitatively demonstrate how our method learns different representations for different visual hierarchies and scenes, which has not been demonstrated in the previous methods. Above previous testing  datasets mostly consist of iconic landmarks or images taken from social media, which makes the dataset a simple memory task, or makes it biased towards certain places. To address this issue we introduce a much harder testing dataset, Google-World-Streets-15k, comprised of images taken from Google Streetview covering the whole planet and present state of the art results. Our code can be found at https://github.com/AHKerrigan/GeoGuessNet.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2023,

title = {3DMODT: Attention-Guided Affinities for Joint Detection & Tracking in 3D Point Clouds},

author = {Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/pdf/2211.00746.pdf},

year  = {2023},

date = {2023-05-29},

urldate = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}