Publications

@conference{Arul2024,

title = {When, What, and with Whom to Communicate: Enhancing RL-based Multi-Robot Navigation through Selective Communication},

author = {S. H. Arul and A. S. Bedi and D. Manocha},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shek2024,

title = {LANCAR: Leveraging Language for Context-Aware Robot Locomotion in Unstructured Environments},

author = {C. L. Shek and X. Wu and W. A. Suttle and C. Busart and E. Zaroukian and D. Manocha and P. Tokekar and A. S. Bedi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2310.00481v2.pdf},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024b,

title = {TrustNavGPT: Trust-Driven Audio-Guided Robot Navigation under Uncertainty with Large Language Models},

author = {Xingpeng Sun and Yiran Zhang and Xindi Tang and Amrit Singh Bedi and Aniket Bera},

url = {https://xingpengsun0.github.io/trustnav/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS) [Oral]},

abstract = {Large language models (LLMs) exhibit a wide range of promising capabilities -- from step-by-step planning to commonsense reasoning --that provide utility for robot navigation. However, as humans communicate with robots in the real world, ambiguity and uncertainty may be embedded inside spoken instructions. While LLMs are proficient at processing text in human conversations, they often encounter difficulties with the nuances of verbal instructions and, thus, remain prone to hallucinate trust in human command. In this work, we present TrustNavGPT, an LLM-based audio-guided navigation agent that uses affective cues in spoken communication—elements such as tone and inflection that convey meaning beyond words—allowing it to assess the trustworthiness of human commands and make effective, safe decisions.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {GDM-Net: gas distribution mapping with a mobile robot using deep reinforcement learning and gaussian process regression},

author = {Iliya Kulbaka and Ayan Dutta and O. Patrick Kreidl and Ladislau Bölöni and Swapnoneel Roy},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2024,

title = {Sparse Points to Dense Clouds: Enhancing 3D Detection with Limited LiDAR Data},

author = {Aakash Kumar and Chen Chen and Ajmal Mian and Neils Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2404.06715v1.pdf

https://aakashjuseja-aj.github.io/Sparse_to_Dense/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

abstract = {3D detection is a critical task that enables machines to identify and locate objects in three-dimensional space. It has a broad range of applications in several fields, including autonomous driving, robotics and augmented reality. Monocular 3D detection is attractive as it requires only a single camera, however, it lacks the accuracy and robustness required for real world applications. High resolution LiDAR on the other hand, can be expensive and lead to interference problems in heavy traffic given their active transmissions. We propose a balanced approach that combines the advantages of monocular and point cloud-based 3D detection. Our method requires only a small number of 3D points, that can be obtained from a low-cost, low-resolution sensor. Specifically, we use only 512 points, which is just 1% of a full LiDAR frame in the KITTI dataset. Our method reconstructs a complete 3D point cloud from this limited 3D information combined with a single image. The reconstructed 3D point cloud and corresponding image can be used by any multi-modal off-the-shelf detector for 3D object detection. By using the proposed network architecture with an off-the-shelf multi-modal 3D detector, the accuracy of 3D detection improves by 20% compared to the state-of-the-art monocular detection methods and 6% to 9% compare to the baseline multi-modal methods on KITTI and JackRabbot datasets.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar

Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf

https://prakashchhipa.github.io/projects/mpd},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method  outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200

},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf

https://daveishan.github.io/finepsuedo-webpage/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias,  classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48,  FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the  robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf

https://arxiv.org/abs/2407.13851

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.

},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and benjamin Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024b,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf

https://daveishan.github.io/avr-webpage/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf

https://arxiv.org/pdf/2407.04370

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tai2024,

title = {uCAP: An Unsupervised Prompting Method for Vision-Language Models},

author = {Kai Sheng Tai and Sirius Chen and Satya Narayan Shukla and Hanchao Yu and Philip Torr and Taipeng Tian and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses a significant limitation that prevents Contrastive Language-Image Pretrained Models (CLIP) from achieving optimal performance on downstream image classification tasks. The key problem with CLIP-style zero-shot classification is that it requires domain-specific context in the form of prompts to better align the class descriptions to the downstream data distribution. In particular, prompts for vision-language models are domain-level texts (e.g., “a centered satellite image of ...”) which, together with the class names, are fed into the text encoder to provide more context for the downstream dataset. These prompts are typically manually tuned, which is time consuming and often sub-optimal. To overcome this bottleneck, this paper proposes uCAP, a method to automatically learn domain-specific prompts/contexts using only unlabeled in-domain images. We achieve this by modeling the generation of images given the class names and a domain-specific prompt with an unsupervised likelihood distribution, and then performing inference of the prompts. We validate the proposed method across various models and datasets, showing that uCAP consistently outperforms manually tuned prompts and related baselines on the evaluated datasets: ImageNet, CIFAR-10, CIFAR-100, OxfordPets (up to 2%), SUN397 (up to 5%), and Caltech101 (up to 3%).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024b,

title = {Fast Encoding and Decoding for Implicit Video Representation},

author = {Hao Chen and Saining Xie and Ser-Nam Lim and Abhinav Shrivastava},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jang2024,

title = {Spherical Linear Interpolation and Text-Anchoring for Zero-shot Composed Retrieval},

author = {Young Kyun Jang and Dat Huynh and Ashish Shah and Wen-Kai Chen and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024,

title = {Towards Multi-modal Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Aritra Dutta and Xin Li and Chen Chen},

url = {https://arxiv.org/pdf/2404.12467.pdf

https://github.com/imguangyu/FedCola},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2024,

title = {ControlNet++: Improving Conditional Controls with Efficient Consistency Feedback},

author = {Ming Li and Taojiannan Yang and Huafeng Kuang and Jie Wu and Zhaoning Wang and Xuefeng Xiao and Chen Chen},

url = {https://arxiv.org/pdf/2404.07987.pdf

https://liming-ai.github.io/ControlNet_Plus_Plus/

https://github.com/liming-ai/ControlNet_Plus_Plus

https://huggingface.co/spaces/limingcv/ControlNet-Plus-Plus},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {To enhance the controllability of text-to-image diffusion models, existing efforts like ControlNet incorporated image-based conditional controls. In this paper, we reveal that existing methods still face significant challenges in generating images that align with the image conditional controls. To this end, we propose ControlNet++, a novel approach that improves controllable generation by explicitly optimizing pixel-level cycle consistency between generated images and conditional controls. Specifically, for an input conditional control, we use a pre-trained discriminative reward model to extract the corresponding condition of the generated images, and then optimize the consistency loss between the input conditional control and extracted condition. A straightforward implementation would be generating images from random noises and then calculating the consistency loss, but such an approach requires storing gradients for multiple sampling timesteps, leading to considerable time and memory costs. To address this, we introduce an efficient reward strategy that deliberately disturbs the input images by adding noise, and then uses the single-step denoised images for reward fine-tuning. This avoids the extensive costs associated with image sampling, allowing for more efficient reward fine-tuning. Extensive experiments show that ControlNet++ significantly improves controllability under various conditional controls. For example, it achieves improvements over ControlNet by 7.9% mIoU, 13.4% SSIM, and 7.6% RMSE, respectively, for segmentation mask, line-art edge, and depth conditions.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {BAMM: Bidirectional Autoregressive Motion Model},

author = {Ekkasit Pinyoanuntapong and Muhammad Usama Saleem and Pu Wang and Minwoo Lee and Srijan Das and Chen Chen},

url = {https://arxiv.org/pdf/2403.19435.pdf

https://exitudio.github.io/BAMM-page/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Generating human motion from text has been dominated by denoising motion models either through diffusion or generative masking process. However, these models face great limitations in usability by requiring prior knowledge of the motion length. Conversely, autoregressive motion models address this limitation by adaptively predicting motion endpoints, at the cost of degraded generation quality and editing capabilities. To address these challenges, we propose Bidirectional Autoregressive Motion Model (BAMM), a novel text-to-motion generation framework. BAMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into discrete tokens in latent space, and (2) a masked self-attention transformer that autoregressively predicts randomly masked tokens via a hybrid attention masking strategy. By unifying generative masked modeling and autoregressive modeling, BAMM captures rich and bidirectional dependencies among motion tokens, while learning the probabilistic mapping from textual inputs to motion outputs with dynamically-adjusted motion sequence length. This feature enables BAMM to simultaneously achieving high-quality motion generation with enhanced usability and built-in motion editability. Extensive experiments on HumanML3D and KIT-ML datasets demonstrate that BAMM surpasses current state-of-the-art methods in both qualitative and quantitative measures.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024,

title = {3DEgo: 3D Editing on the Go!},

author = {Umar Khalid and Hasan Iqbal and Azib Farooq and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2407.10102

https://3dego.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {We introduce 3DEgo to address a novel problem of directly synthesizing photorealistic 3D scenes from monocular videos guided by textual prompts. Conventional methods construct a text-conditioned 3D scene through a three-stage process, involving pose estimation using Structure-from-Motion (SfM) libraries like COLMAP, initializing the 3D model with unedited images, and iteratively updating the dataset with edited images to achieve a 3D scene with text fidelity. Our framework streamlines the conventional multi-stage 3D editing process into a single-stage workflow by overcoming the reliance on COLMAP and eliminating the cost of model initialization. We apply a diffusion model to edit video frames prior to 3D scene creation by incorporating our designed noise blender module for enhancing multi-view editing consistency, a step that does not require additional training or fine-tuning of T2I diffusion models. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the multi-view consistent edited frames, capitalizing on the inherent temporal continuity and explicit point cloud data. 3DEgo demonstrates remarkable editing precision, speed, and adaptability across a variety of video sources, as validated by extensive evaluations on six datasets, including our own prepared GS25 dataset.



},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024b,

title = {LatentEditor: Text Driven Local Editing of 3D Scenes},

author = {Umar Khalid and Hasan Iqbal and Muhammad Tayyab and Md Nazmul Karim and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2312.09313.pdf

https://latenteditor.github.io/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {While neural fields have made significant strides in view synthesis and scene reconstruction, editing them poses a formidable challenge due to their implicit encoding of geometry and texture information from multi-view inputs. In this paper, we introduce LatenEditor, an innovative framework designed to empower users with the ability to perform precise and locally controlled editing of neural fields using text prompts. Leveraging denoising diffusion models, we successfully embed real-world scenes into the latent space, resulting in a faster and more adaptable NeRF backbone for editing compared to traditional methods. To enhance editing precision, we introduce a delta score to calculate the 2D mask in the latent space that serves as a guide for local modifications while preserving irrelevant regions. Our novel pixel-level scoring approach harnesses the power of InstructPix2Pix (IP2P) to discern the disparity between IP2P conditional and unconditional noise predictions in the latent space. The edited latents conditioned on the 2D masks are then iteratively updated in the training set to achieve 3D local editing. Our approach achieves faster editing speeds and superior output quality compared to existing 3D editing models, bridging the gap between textual instructions and high-quality 3D scene editing in latent space.



},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Karim2024,

title = {Free-Editor: Zero-shot Text-driven 3D Scene Editing},

author = {Md Nazmul Karim and Hasan Iqbal and Umar Khalid and Chen Chen and Jing Hua

},

url = {https://arxiv.org/pdf/2312.13663.pdf

https://free-editor.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Text-to-Image (T2I) diffusion models have gained popularity recently due to their multipurpose and easy-to-use nature, e.g. image and video generation as well as editing. However, training a diffusion model specifically for 3D scene editing is not straightforward due to the lack of large-scale datasets. To date, editing 3D scenes requires either re-training the model to adapt to various 3D edited scenes or design-specific methods for each special editing type. Furthermore, state-of-the-art (SOTA) methods require multiple synchronized edited images from the same scene to facilitate the scene editing. Due to the current limitations of T2I models, it is very challenging to apply consis- tent editing effects to multiple images, i.e. multi-view inconsistency in editing. This in turn compromises the desired 3D scene editing performance if these images are used. In our work, we propose a novel training-free 3D scene editing technique, FREE-EDITOR, which allows users to edit 3D scenes without further re-training the model during test time. Our proposed method successfully avoids the multi- view style inconsistency issue in SOTA methods with the help of a “single-view editing” scheme. Specifically, we show that editing a particular 3D scene can be performed by only modifying a single view. To this end, we introduce an Edit Transformer that enforces intra-view consistency and inter-view style transfer by utilizing self- and cross- attention, respectively. Since it is no longer required to re-train the model and edit every view in a scene, the editing time, as well as memory resources, are reduced significantly, e.g., the runtime being ∼ 20× faster than SOTA. We have conducted extensive experiments on a wide range of benchmark datasets and achieve diverse editing capabilities with our proposed technique.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fang2024,

title = {Rethinking Weakly-supervised Video Temporal Grounding From a Game Perspective},

author = {Xiang Fang and Zeyu Xiong and Wanlong Fang and Xiaoye Qu and Chen Chen and Jianfeng Dong and Keke Tang and Pan Zhou and Yu Cheng and Daizong Liu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ECCV2024_Grounding_camera.pdf

https://eccv2024.ecva.net/virtual/2024/poster/1833},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses the challenging task of weakly-supervised video temporal grounding. Existing approaches are generally based on the moment candidate selection pipeline that utilizes contrastive learning and reconstruction paradigm for scoring the pre-defined moments. Although they have achieved significant progress, we argue that their current frameworks have overlooked two indispensable issues: (1) Coarse-grained cross-modal learning: previous methods solely capture the global video-level alignment with the query, failing to model the detailed consistency between video frames and query words for accurately grounding the moment boundaries. (2) Complex moment candidates: the performance of these methods severely relies on the quality of moment candidates, which are also time-consuming and complicated for selection. To this end, in this paper, we make the first attempt to tackle this task from a novel game perspective, which effectively learns the uncertain relationship between each frame-word pair with diverse granularity and flexible combination for fine-grained cross-modal interaction. Specifically, we creatively model each video frame and query word as game players with multivariate cooperative game theory to learn their contribution to the cross-modal similarity score. By quantifying the trend of frame-word cooperation within a coalition via the game-theoretic interaction, we are able to value all uncertain but possible correspondence between frames and words. At last, instead of using moment proposals, we utilize the learned query-guided frame-wise scores for fine-grained moment boundary grounding. Experiments show that our method achieves superior performance on both Charades-STA and ActivityNet Caption datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf

https://arxiv.org/abs/2408.02840

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf

https://parthpk.github.io/cityguessr-webpage/

},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {Workshop on Computer Vision For Videogames (CV2)},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38]. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Augmented Neural Fine-tuning for Efficient Backdoor Purification},

author = {Nazmul Karim and Abdullah Al Arafat and Umar Khalid and Zhishan Guo and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2024_ECCV_NFT.pdf

https://arxiv.org/pdf/2407.10052

https://github.com/nazmul-karim170/NFT-Augmented-Backdoor-Purification},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent studies have revealed the vulnerability of deep neural networks (DNNs) to various backdoor attacks, where the behavior of DNNs can be compromised by utilizing certain types of triggers or poisoning mechanisms. State-of-the-art (SOTA) defenses employ too sophisticated mechanisms that require either a computationally expensive adversarial search module for reverse-engineering the trigger distribution or an over-sensitive hyper-parameter selection module. Moreover, they offer sub-par performance in challenging scenarios, e.g., limited validation data and strong attacks. In this paper, we propose—Neural mask Fine-Tuning (NFT)—with an aim to optimally re-organize the neuron activities in a way that the effect of the backdoor is removed. Utilizing a simple data augmentation like MixUp, NFT relaxes the trigger synthesis process and eliminates the requirement of the adversarial search module. Our study further reveals that direct weight fine-tuning under limited validation data results in poor post-purification clean test accuracy, primarily due to overfitting issue. To overcome this, we propose to fine-tune neural masks instead of model weights. In addition, a mask regularizer

has been devised to further mitigate the model drift during the purification process. The distinct characteristics of NFT render it highly efficient in both runtime and sample usage, as it can remove the backdoor even when a single sample is available from each class. We validate the effectiveness of NFT through extensive experiments covering the tasks of image classification, object detection, video action recognition, 3D point cloud, and natural language processing. We evaluate our method against 14 different attacks (LIRA, WaNet, etc.) on 11 benchmark data sets (ImageNet, UCF101, Pascal VOC, ModelNet, OpenSubtitles2012, etc.). },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2024,

title = {A Dual-Augmentor Framework for Domain Generalization in 3D Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://arxiv.org/pdf/2403.11310.pdf

https://github.com/davidpengucf/DAF-DG

},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {3D human pose data collected in controlled laboratory settings present challenges for pose estimators that generalize across diverse scenarios. To address this, domain generalization is employed. Current methodologies in domain generalization for 3D human pose estimation typically utilize adversarial training to generate synthetic poses for training. Nonetheless, these approaches exhibit several limitations. First, the lack of prior information about the target domain complicates the application of suitable augmentation through a single pose augmentor, affecting generalization on target domains. Moreover, adversarial training’s discriminator tends to enforce similarity between source and synthesized poses, impeding the exploration of out-ofsource distributions. Furthermore, the pose estimator’s optimization is not exposed to domain shifts, limiting its overall generalization ability. 

 

To address these limitations, we propose a novel framework featuring two pose augmentors: the weak and the strong augmentors. Our framework employs differential strategies for generation and discrimination processes, facilitating the preservation of knowledge related to source poses and the exploration of out-of-source distributions without prior information about target poses. Besides, we leverage meta-optimization to simulate domain shifts in the optimization process of the pose estimator, thereby improving its generalization ability. Our proposed approach significantly outperforms existing methods, as demonstrated through comprehensive experiments on various benchmark datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2024,

title = {Multi-View Attentive Contextualization for Multi-View 3D Object Detection},

author = {Xianpeng Liu and Ce Zheng and Ming Qian and Nan Xue and Chen Chen and Zhebin Zhang and Chen Li and Tianfu Wu},

year  = {2024},

date = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yuan2024,

title = {Towards Surveillance Video-and-Language Understanding: New Dataset, Baselines, and Challenges},

author = {Tongtong Yuan and Xuange Zhang and Kun Liu and Bo Liu and Chen Chen and Jian Jin and Zhenzhen Jiao},

url = {https://arxiv.org/pdf/2309.13925.pdf

https://xuange923.github.io/Surveillance-Video-Understanding

https://github.com/Xuange923/Surveillance-Video-Understanding},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Surveillance videos are an essential component of daily life with various critical applications, particularly in public security. However, current surveillance video tasks mainly focus on classifying and localizing anomalous events. Existing methods are limited to detecting and classifying the predefined events with unsatisfactory semantic understanding, although they have obtained considerable performance. To address this issue, we propose a new research direction of surveillance video-and-language understanding, and construct the first multimodal surveillance video dataset. We manually annotate the real-world surveillance dataset UCF-Crime with fine-grained event content and timing. Our newly annotated dataset, UCA UCF-Crime Annotation), contains 23,542 sentences, with an average length of 20 words, and its annotated videos are as long as 110.7 hours. Furthermore, we benchmark SOTA models for four multimodal tasks on this newly created dataset, which serve as new baselines for surveillance video-and-language understanding. Through our experiments, we find that mainstream models used in previously publicly available datasets perform poorly on surveillance video, which demonstrates the new challenges in surveillance video-and-language understanding. To validate the effectiveness of our UCA, we conducted experiments on multimodal anomaly detection. The results demonstrate that our multimodal surveillance learning can improve the performance of conventional anomaly detection tasks. All the experiments highlight the necessity of constructing this dataset to advance surveillance AI.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pinyoanuntapong2024,

title = {MMM: Generative Masked Motion Model},

author = {Ekkasit Pinyoanuntapong and Pu Wang and Minwoo Lee and Chen Chen},

url = {https://arxiv.org/pdf/2312.03596.pdf

https://exitudio.github.io/MMM-page/

https://github.com/exitudio/MMM/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in text-to-motion generation using diffusion and autoregressive models have shown promising results. However, these models often suffer from a trade-off between real-time performance, high fidelity, and motion editability. To address this gap, we introduce MMM, a novel yet simple motion generation paradigm based on Masked Motion Model. MMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into a sequence of discrete tokens in latent space, and (2) a conditional masked motion transformer that learns to predict randomly masked motion tokens, conditioned on the pre-computed text tokens. By attending to motion and text tokens in all directions, MMM explicitly captures inherent dependency among motion tokens and semantic mapping between motion and text tokens. During inference, this allows parallel and iterative decoding of multiple motion tokens that are highly consistent with fine-grained text descriptions, therefore simultaneously achieving high-fidelity and high-speed motion generation. In addition, MMM has innate motion editability. By simply placing mask tokens in the place that needs editing, MMM automatically fills the gaps while guaranteeing smooth transitions between editing and non-editing parts. Extensive experiments on the HumanML3D and KIT-ML datasets demonstrate that MMM surpasses current leading methods in generating high-quality motion (evidenced by superior FID scores of 0.08 and 0.429), while offering advanced editing features such as body-part modification, motion in-betweening, and the synthesis of long motion sequences. In addition, MMM is two orders of magnitude faster on a single mid-range GPU than editable motion diffusion models.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024,

title = {Skeleton-in-Context: Unified Skeleton Sequence Modeling with In-Context Learning},

author = {Xinshun Wang and Zhongbin Fang and Xia Li, Xiangtai Li and Chen Chen and Mengyuan Liu},

url = {https://arxiv.org/pdf/2312.03703.pdf

https://bradleywang0416.github.io/skeletonincontext/

https://github.com/fanglaosi/Skeleton-in-Context},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In-context learning provides a new perspective for multi-task modeling for vision and NLP. Under this setting, the model can perceive tasks from prompts and accomplish them without any extra task-specific head predictions or model finetuning. However, Skeleton sequence modeling via in-context learning remains unexplored. Directly applying existing in-context models from other areas onto skeleton sequences fails due to the inter-frame and cross-task pose similarity that makes it outstandingly hard to perceive the task correctly from a subtle context.



To address this challenge, we propose Skeleton-in-Context (SiC), an effective framework for in-context skeleton sequence modeling. Our SiC is able to handle multiple skeleton-based tasks simultaneously after a single training process and accomplish each task from context according to the given prompt. It can further generalize to new, unseen tasks according to customized prompts. To facilitate context perception, we additionally propose a task-unified prompt, which adaptively learns tasks of different natures, such as partial joint-level generation, sequence-level prediction, or 2D-to-3D motion prediction. We conduct extensive experiments to evaluate the effectiveness of our SiC on multiple tasks, including motion prediction, pose estimation, joint completion, and future pose estimation. We also evaluate its generalization capability on unseen tasks such as motion-in-between. These experiments show that our model achieves state-of-the-art multi-task performance and even outperforms single-task methods on certain tasks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024,

title = {OST: Refining Text Knowledge with Optimal Spatio-Temporal Descriptor for General Video Recognition},

author = {Tongjia Chen and Hongshan Yu and Zhengeng Yang and Zechuan Li and Wei Sun and Chen Chen},

url = {https://arxiv.org/pdf/2312.00096.pdf

https://tomchen-ctj.github.io/OST/

https://github.com/tomchen-ctj/OST},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Due to the resource-intensive nature of training vision-language models on expansive video data, a majority of studies have centered on adapting pre-trained image-language models to the video domain. Dominant pipelines propose to tackle the visual discrepancies with additional temporal learners while overlooking the substantial discrepancy for web-scaled descriptive narratives and concise action category names, leading to less distinct semantic space and potential performance limitations. In this work, we prioritize the refinement of text knowledge to facilitate generalizable video recognition. To address the limitations of the less distinct semantic space of category names, we prompt a large language model (LLM) to augment action class names into Spatio-Temporal Descriptors thus bridging the textual discrepancy and serving as a knowledge base for general recognition. Moreover, to assign the best descriptors with different video instances, we propose Optimal Descriptor Solver, forming the video recognition problem as solving the optimal matching flow across frame-level representations and descriptors. Comprehensive evaluations in zero-shot, few-shot, and fully supervised video recognition highlight the effectiveness of our approach. Our best model achieves a state-of-the-art zero-shot accuracy of 75.1% on Kinetics-600.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2024,

title = {VidLA: Video-Language Alignment at Scale},

author = {Mamshad Nayeem Rizve and Fan Fei and Jayakrishnan Unnikrishnan and Son Tran and Benjamin Z. Yao and Belinda Zeng and Mubarak Shah and Trishul Chilimbi},

url = {https://arxiv.org/abs/2403.14870},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {In this paper, we propose VidLA, an approach for video-language alignment at scale. There are two major limitations of previous video-language alignment approaches. First, they do not capture both short-range and long-range temporal dependencies and typically employ complex hierarchical deep network architectures that are hard to integrate with existing pretrained image-text foundation models. To effectively address this limitation, we instead keep the network architecture simple and use a set of data tokens that operate at different temporal resolutions in a hierarchical manner, accounting for the temporally hierarchical nature of videos. By employing a simple two-tower architecture, we are able to initialize our video-language model with pretrained image-text foundation models, thereby boosting the final performance. Second, existing video-language alignment works struggle due to the lack of semantically aligned large-scale training data. To overcome it, we leverage recent LLMs to curate the largest video-language dataset to date with better visual grounding. Furthermore, unlike existing video-text datasets which only contain short clips, our dataset is enriched with video clips of varying durations to aid our temporally hierarchical data tokens in extracting better representations at varying temporal scales. Overall, empirical results show that our proposed approach surpasses state-of-the-art methods on multiple retrieval benchmarks, especially on longer videos, and performs competitively on classification benchmarks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ristea2024,

title = {Self-Distilled Masked Auto-Encoders are Efficient Video Anomaly Detectors},

author = {Nicolae Catalin Ristea and Florinel Alin Croitoru and Radu Tudor Ionescu and Marius Popescu and Fahad Khan and Mubarak Shah},

url = {https://arxiv.org/abs/2306.12041

https://github.com/ristea/aed-mae},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {We propose an efficient abnormal event detection model based on a lightweight masked auto-encoder (AE) applied at the video frame level. The novelty of the proposed model is threefold. First, we introduce an approach to weight tokens based on motion gradients, thus shifting the focus from the static background scene to the foreground objects. Second, we integrate a teacher decoder and a student decoder into our architecture, leveraging the discrepancy between the outputs given by the two decoders to improve anomaly detection. Third, we generate synthetic abnormal events to augment the training videos, and task the masked AE model to jointly reconstruct the original frames (without anomalies) and the corresponding pixel-level anomaly maps. Our design leads to an efficient and effective model, as demonstrated by the extensive experiments carried out on four benchmarks: Avenue, ShanghaiTech, UBnormal and UCSD Ped2. The empirical results show that our model achieves an excellent trade-off between speed and accuracy, obtaining competitive AUC scores, while processing 1655 FPS. Hence, our model is between 8 and 70 times faster than competing methods. We also conduct an ablation study to justify our design. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2024,

title = {Composed Video Retrieval via Enriched Context and Discriminative Embeddings},

author = {Omkar Chakradhar Thawakar and Muzammal Naseer and Rao Muhammad Anwer and Salman Khan and Michael Felsberg and Mubarak Shah and Fahad Khan },

url = {https://arxiv.org/abs/2403.16997

https://github.com/OmkarThawakar/composed-video-retrieval},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Composed video retrieval (CoVR) is a challenging problem in computer vision which has recently highlighted the integration of modification text with visual queries for more sophisticated video search in large databases. Existing works predominantly rely on visual queries combined with modification text to distinguish relevant videos. However, such a strategy struggles to fully preserve the rich query-specific context in retrieved target videos and only represents the target video using visual embedding. We introduce a novel CoVR framework that leverages detailed language descriptions to explicitly encode query-specific contextual information and learns discriminative embeddings of vision only, text only and vision-text for better alignment to accurately retrieve matched target videos. Our proposed framework can be flexibly employed for both composed video (CoVR) and image (CoIR) retrieval tasks. Experiments on three datasets show that our approach obtains state-of-the-art performance for both CovR and zero-shot CoIR tasks, achieving gains as high as around 7% in terms of recall@K=1 score.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dutta2024,

title = {Multiview Aerial Visual RECognition (MAVREC) Dataset: Can Multi-view Improve Aerial Visual Perception?},

author = {Aritra Dutta and Srijan Das and Jacob Nielsen and Rajatsubhra Chakraborty and Mubarak Shah},

url = {https://arxiv.org/abs/2312.04548

https://mavrec.github.io/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Despite the commercial abundance of UAVs, aerial data acquisition remains challenging, and the existing Asia and North America-centric open-source UAV datasets are small-scale or low-resolution and lack diversity in scene contextuality. Additionally, the color content of the scenes, solar-zenith angle, and population density of different geographies influence the data diversity. These two factors conjointly render suboptimal aerial-visual perception of the deep neural network (DNN) models trained primarily on the ground-view data, including the open-world foundational models.



To pave the way for a transformative era of aerial detection, we present Multiview Aerial Visual RECognition or MAVREC, a video dataset where we record synchronized scenes from different perspectives -- ground camera and drone-mounted camera. MAVREC consists of around 2.5 hours of industry-standard 2.7K resolution video sequences, more than 0.5 million frames, and 1.1 million annotated bounding boxes. This makes MAVREC the largest ground and aerial-view dataset, and the fourth largest among all drone-based datasets across all modalities and tasks. Through our extensive benchmarking on MAVREC, we recognize that augmenting object detectors with ground-view images from the corresponding geographical location is a superior pre-training strategy for aerial detection. Building on this strategy, we benchmark MAVREC with a curriculum-based semi-supervised object detection approach that leverages labeled (ground and aerial) and unlabeled (only aerial) images to enhance the aerial detection.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2024,

title = {Activity-Biometrics: Person Identification from Daily Activities},

author = {Shehreen Azad, and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2403.17360

https://github.com/sacrcv/Activity-Biometrics/

},

year  = {2024},

date = {2024-06-17},

abstract = {In this work, we study a novel problem which focuses on person identification while performing daily activities. Learning biometric features from RGB videos is challenging due to spatio-temporal complexity and presence of appearance biases such as clothing color and background. We propose ABNet, a novel framework which leverages disentanglement of biometric and non-biometric features to perform effective person identification from daily activities. ABNet relies on a bias-less teacher to learn biometric features from RGB videos and explicitly disentangle non-biometric features with the help of biometric distortion. In addition, ABNet also exploits activity prior for biometrics which is enabled by joint biometric and activity learning. We perform comprehensive evaluation of the proposed approach across five different datasets which are derived from existing activity recognition benchmarks. Furthermore, we extensively compare ABNet with existing works in person identification and demonstrate its effectiveness for activity-based biometrics across all five datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Frakes2024,

title = {Efficient and consistent zero-shot video generation with diffusion models},

author = {Ethan Frakes and Umar Khalid and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Report-Ethan-Frakes.pdf

https://spie.org/defense-commercial-sensing/presentation/Efficient-and-consistent-zero-shot-video-generation-with-diffusion-models/13034-8#_=_},

year  = {2024},

date = {2024-04-22},

urldate = {2024-04-22},

publisher = {SPIE Defense+ Commercial Sensing},

abstract = {Recent diffusion-based generative models employ methods such as one-shot fine-tuning an image diffusion model for video generation. However, this leads to long video generation times and suboptimal efficiency. To resolve this long generation time, zero-shot text-to-video models eliminate the fine-tuning method entirely and can generate novel videos from a text prompt alone. While the zero-shot generation method greatly reduces generation time, many models rely on inefficient cross-frame attention processors, hindering the diffusion model’s utilization for real-time video generation. We address this issue by introducing more efficient attention processors to a video diffusion model. Specifically, we use attention processors (i.e. xFormers, FlashAttention, and HyperAttention) that are highly optimized for efficiency and hardware parallelization. We then apply these processors to a video generator and test with both older diffusion models such as Stable Diffusion 1.5 and newer, high-quality models such as Stable Diffusion XL. Our results show that using efficient attention processors alone can reduce generation time by around 25%, while not resulting in any change in video quality. Combined with the use of higher quality models, this use of efficient attention processors in zero-shot generation presents a substantial efficiency and quality increase, greatly expanding the video diffusion model’s application to real-time video generation.},

keywords = {REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cepeda2023,

title = {GeoCLIP: Clip-Inspired Alignment between Locations and Images for Effective Worldwide Geo-localization},

author = {Vicente Vivanco Cepeda and Gaurav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_paper.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_supplementary.pdf

https://vicentevivan.github.io/GeoCLIP/},

year  = {2023},

date = {2023-12-11},

publisher = {Thirty-seventh Conference on Neural Information Processing Systems},

abstract = {Worldwide Geo-localization aims to pinpoint the precise location of images taken anywhere on Earth. This task has considerable challenges due to immense variation in geographic landscapes. The image-to-image retrieval-based approaches fail to solve this problem on a global scale as it is not feasible to construct a large gallery of images covering the entire world. Instead, existing approaches divide the globe into discrete geographic cells, transforming the problem into a classification task. However, their performance is limited by the predefined classes and often results in inaccurate localizations when an image’s location significantly deviates from its class center. To overcome these limitations, we propose GeoCLIP, a novel CLIP-inspired Image-to-GPS retrieval approach that enforces alignment between the image and its corresponding GPS locations. GeoCLIP’s location encoder models the Earth as a continuous function by employing positional encoding through random Fourier features and constructing a hierarchical representation that captures information at varying resolutions to yield a semantically rich highdimensional feature suitable to use even beyond geo-localization. To the best of our knowledge, this is the first work employing GPS encoding for geo-localization. We demonstrate the efficacy of our method via extensive experiments and ablations on benchmark datasets. We achieve competitive performance with just 20% of training data, highlighting its effectiveness even in limited-data settings. Furthermore, we qualitatively demonstrate geo-localization using a text query by leveraging CLIP backbone of our image encoder.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2023,

title = {On Occlusions in Video Action Detection: Benchmark Datasets And Training Recipes},

author = {Rajat Modi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/nips_23.pdf},

year  = {2023},

date = {2023-12-10},

publisher = {NeurIPS 2023.},

abstract = {This paper explores the impact of occlusions in video action detection. We facilitate this study by introducing five new benchmark datasets namely O-UCF and OJHMDB consisting of synthetically controlled static/dynamic occlusions, OVISUCF and OVIS-JHMDB consisting of occlusions with realistic motions and Real-OUCF for occlusions in realistic-world scenarios. We formally confirm an intuitive expectation: existing models suffer a lot as occlusion severity is increased and exhibit different behaviours when occluders are static vs when they are  moving. We discover several curious phenomenon emerging in neural nets: 1) transformers  can naturally outperform CNN models which might have even used occlusion as a form of data augmentation during training 2) incorporating symbolic-components like capsules to such backbones allows them to bind to occluders never even seen during training and 3) Islands of agreement (similar to the ones hypothesized in Hinton et Al’s GLOM) can emerge in realistic images/videos without instance-level supervision, distillation or contrastive-based objectives2(eg. video-textual training). Such emergent properties allow us to derive simple yet effective training recipes which lead to robust occlusion models inductively satisfying the first two stages of the binding mechanism (grouping/segregation). Models leveraging these recipes outperform existing video action-detectors under occlusion by 32.3% on O-UCF, 32.7% on O-JHMDB & 2.6% on Real-OUCF in terms of the vMAP metric.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Frequency Domain Adversarial Training for Robust Volumetric Medical Segmentation},

author = {Asif Hanif and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Frequency-Domain-Adversarial-Training-for-Robust-Volumetric-Medical-Segmentation.pdf

https://github.com/asif-hanif/vafa},

doi = {https://doi.org/10.48550/arXiv.2307.07269},

year  = {2023},

date = {2023-10-08},

publisher = {The 26th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2023},

keywords = {MICCAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2023,

title = {AlignDet: Aligning Pre-training and Fine-tuning in Object Detection},

author = {Ming Li and Jie Wu and Xionghui Wang and Chen Chen and Jie Qin and Xuefeng Xiao and Rui Wang and Min Zheng and Xin Pan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.11077.pdf

https://arxiv.org/abs/2307.11077

https://github.com/liming-ai/AlignDet

https://openreview.net/forum?id=8PA2nX9v_r2

https://liming-ai.github.io/AlignDet/},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2023,

title = {Source-free Domain Adaptive Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.03202.pdf

https://arxiv.org/abs/2308.03202

https://github.com/davidpengucf/SFDAHPE},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Human Pose Estimation (HPE) is widely used in various fields, including motion analysis, healthcare, and virtual reality. However, the great expenses of labeled real-world datasets present a significant challenge for HPE. To  overcome this, one approach is to train HPE models on synthetic datasets and then perform domain adaptation (DA) on realworld data. Unfortunately, existing DA methods for HPE neglect data privacy and security by using both source and target data in the adaptation process. To this end, we propose a new task, named sourcefree domain adaptive HPE, which aims to address the challenges of cross-domain learning of HPE without access to source data during the adaptation process. We further propose a novel framework that consists of three models: source model, intermediate model, and target model, which explores the task from both sourceprotect and target-relevant perspectives. The sourceprotect module preserves source information more effectively while resisting noise, and the target-relevant module reduces the sparsity of spatial representations by building a novel spatial probability space, and pose-specific contrastive learning and information maximization are proposed on the basis of this space. Comprehensive experiments on several domain adaptive HPE benchmarks show that the proposed method outperforms existing approaches by a considerable margin. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {FedPerfix: Towards Partial Model Personalization of Vision Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.09160.pdf

https://arxiv.org/abs/2308.09160

https://github.com/imguangyu/FedPerfix},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We propose and analyze a general framework of federated learning with  partial model personalization. Compared with full model personalization, partial model personalization relies on domain knowledge to select a small portion of the model to personalize, thus imposing a much smaller on-device memory footprint. We propose two federated optimization algorithms for training partially personalized models, where the shared and personal parameters are updated either simultaneously or alternately on each device, but only the shared parameters are communicated and aggregated at the server. We give convergence analyses of both algorithms for minimizing smooth nonconvex functions, providing theoretical support of them for training deep learning models. Our experiments on real-world image and text datasets demonstrate that (a) partial model personalization can obtain most of the benefit of full model personalization with a small fraction of personalized parameters, and, (b) the alternating update algorithm often outperforms the simultaneous update algorithm.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PGFed: Personalize Each Client's Global Objective for Federated Learning},

author = {Jun Luo and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.01448.pdf

https://github.com/ljaiverson/pgfed},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The mediocre performance of conventional federated learning (FL) over heterogeneous data has been facilitating personalized FL solutions, where, unlike conventional FL which trains a single global consensus model, different models are allowed for different clients. However, in most existing personalized FL algorithms, the collaborative knowledge across the federation was only implicitly passed to the clients in ways such as model aggregation or regularization. We observed that this implicit knowledge transfer fails to maximize the potential value of each client's empirical risk toward other clients. Based on our observation, in this work, we propose Personalized Global Federated Learning (PGFed), a novel personalized FL framework that enables each client to personalize its own global objective by explicitly and adaptively aggregating the empirical risks of itself and other clients. To avoid massive (O(N2)) communication overhead and potential privacy leakage, each client's risk is estimated through a first-order approximation for other clients' adaptive risk aggregation. On top of PGFed, we develop a momentum upgrade, dubbed PGFedMo, to more efficiently utilize clients' empirical risks. Our extensive experiments under different federated settings with benchmark datasets show consistent improvements of PGFed over the compared state-of-the-art alternatives.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {A Large-scale Study of Spatiotemporal Representation Learning with a New Benchmark on Action Recognition},

author = {Andong Dang and Taojiannan Yang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.13505.pdf

https://arxiv.org/abs/2303.13505

https://github.com/AndongDeng/BEAR},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The goal of building a benchmark (suite of datasets) is to provide a unified protocol for fair evaluation and thus facilitate the evolution of a specific area. Nonetheless, we point out that existing protocols of action recognition could yield partial evaluations due to several limitations. To comprehensively probe the effectiveness of spatiotemporal representation learning, we introduce BEAR, a new BEnchmark on video Action Recognition. BEAR is a collection of 18 video datasets grouped into 5 categories (anomaly, gesture, daily, sports, and instructional), which covers a diverse set of real-world applications. With BEAR, we thoroughly evaluate 6 common spatiotemporal models pre-trained by both supervised and self-supervised learning. We also report transfer performance via standard finetuning, few-shot finetuning, and unsupervised domain adaptation. Our observation suggests that current state-of-the-art cannot solidly guarantee high performance on datasets close to real-world applications, and we hope BEAR can serve as a fair and challenging evaluation benchmark to gain insights on building next-generation spatiotemporal learners. Our dataset, code, and models are released at: https://github.com/AndongDeng/BEAR},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Towards Geospatial Foundation Models via Continual Pretraining},

author = {Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2302.04476.pdf

https://arxiv.org/abs/2302.04476

https://github.com/mmendiet/GFM},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Geospatial technologies are becoming increasingly essential in our world for a wide range of applications, including agriculture, urban planning, and disaster response. To help improve the applicability and performance of deep learning models on these geospatial tasks, various works have begun investigating foundation models for this domain. Researchers have explored two prominent approaches for introducing such models in geospatial applications, but both have drawbacks in terms of limited performance benefit or prohibitive training cost. Therefore, in this work, we propose a novel paradigm for building highly effective geospatial foundation models with minimal resource cost and carbon impact. We first construct a compact yet diverse dataset from multiple sources to promote feature diversity, which we term GeoPile. Then, we investigate the potential of continual pretraining from large-scale ImageNet-22k models and propose a multi-objective continual pretraining paradigm, which leverages the strong representations of ImageNet while simultaneously providing the freedom to learn valuable in-domain features. Our approach outperforms previous state-of-the-art geospatial pretraining methods in an extensive evaluation on seven downstream datasets covering various tasks such as change detection, classification, multi-label classification, semantic segmentation, and super-resolution.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Multi-view Self-supervised Disentanglement for General Image Denoising},

author = {Hao Chen and Chenyuan Qu and Yu Zhang and Chen Chen and Jianbo Jiao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Final_Version.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Supplymentary_Final_Version.pdf

https://chqwer2.github.io/MeD/

https://github.com/chqwer2/Multi-view-Self-supervised-Disentanglement-Denoising},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {With its significant performance improvements, the deep

learning paradigm has become a standard tool for modern

image denoisers. While promising performance has been

shown on seen noise distributions, existing approaches often suffer from generalisation to unseen noise types or general and real noise. It is understandable as the model is

designed to learn paired mapping (e.g. from a noisy image

to its clean version). In this paper, we instead propose to

learn to disentangle the noisy image, under the intuitive

assumption that different corrupted versions of the same

clean image share a common latent space. A self-supervised

learning framework is proposed to achieve the goal, without looking at the latent clean image. By taking two different corrupted versions of the same image as input, the proposed Multi-view Self-supervised Disentanglement (MeD) approach learns to disentangle the latent clean features from the corruptions and recover the clean image consequently. Extensive experimental analysis on both synthetic and real noise shows the superiority of the proposed method over prior self-supervised approaches, especially on unseen novel noise types. On real noise, the proposed method even outperforms its supervised counterparts by over 3 dB. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {RenderIH: A large-scale synthetic dataset for 3D interacting hand pose estimation},

author = {Lijun Li, Linrui Tian and Xindi Zhang and Qi Wang and Bang Zhang and Liefeng Bo and Mengyuan Liu and Chen Chen},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023b,

title = {When Do Curricula Work in Federated Learning? },

author = {Saeed Vahidian and Sreevatsank Kadaveru and Woonjoon Baek and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.12712.pdf

https://arxiv.org/abs/2212.12712},

doi = {https://doi.org/10.48550/arXiv.2212.12712},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {An oft-cited open problem of federated learning is the existence of data heterogeneity at the clients. One pathway to understanding the drastic accuracy drop in federated learning is by scrutinizing the behavior of the clients' deep models on data with different levels of "difficulty", which has been left unaddressed. In this paper, we investigate a different and rarely studied dimension of FL: ordered learning. Specifically, we aim to investigate how ordered learning principles can contribute to alleviating the heterogeneity effects in FL. We present theoretical analysis and conduct extensive empirical studies on the efficacy of orderings spanning three kinds of learning: curriculum, anti-curriculum, and random curriculum. We find that curriculum learning largely alleviates non-IIDness. Interestingly, the more disparate the data distributions across clients the more they benefit from ordered learning. We provide analysis explaining this phenomenon, specifically indicating how curriculum training appears to make the objective landscape progressively less convex, suggesting fast converging iterations at the beginning of the training procedure. We derive quantitative results of convergence for both convex and nonconvex objectives by modeling the curriculum training on federated devices as local SGD with locally biased stochastic gradients. Also, inspired by ordered learning, we propose a novel client selection technique that benefits from the real-world disparity in the clients. Our proposed approach to client selection has a synergic effect when applied together with ordered learning in FL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Samarasinghe2023,

title = {CDFSL-V: Cross-Domain Few-Shot Learning for Videos},

author = {Sarinda Samarasinghe and Mamshad Nayeem and Rizve Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CDFSL_Video_Combined_Final.pdf

https://sarinda251.github.io/CDFSL-V-site/

https://www.youtube.com/watch?v=RdlEzfW013o},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Few-shot video action recognition is an effective approach to recognizing new categories with only a few labeled examples, thereby reducing the challenges associated with collecting and annotating large-scale video datasets. Existing methods in video action recognition rely on large labeled datasets from the same domain. However, this setup is not realistic as novel categories may come from different data domains that may have different spatial and temporal characteristics. This dissimilarity between the source and target domains can pose a significant challenge, rendering traditional few-shot action recognition techniques ineffective. To address this issue, in this work, we propose a novel cross-domain few-shot video action recognition method that leverages self-supervised learning and curriculum learning to balance the information from the source and target domains. To be particular, our method employs a masked autoencoder-based self-supervised training objective to learn from both source and target data in a self-supervised manner. Then a progressive curriculum balances learning the discriminative information from the source dataset with the generic information learned from the target domain. Initially, our curriculum utilizes supervised learning to learn class discriminative features from the source data. As the training progresses, we transition to learning target-domain-specific features. We propose a progressive curriculum to encourage the emergence of rich features in the target domain based on class discriminative supervised features in the source domain. %a schedule that helps with this transition. We evaluate our method on several challenging benchmark datasets and demonstrate that our approach outperforms existing cross-domain few-shot learning techniques.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Preserving Modality Structure Improves Multi-Modal Learning },

author = {Swetha Sirnam and Mamshad Nayeem Rizve and Hilde Kuhne and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.13077.pdf

https://arxiv.org/abs/2308.13077

https://github.com/Swetha5/Multi_Sinkhorn_Knopp

https://swetha5.github.io/MultiSK/

https://youtu.be/1CrGkUATy50

},

doi = {https://doi.org/10.48550/arXiv.2308.13077},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Self-supervised learning on large-scale multi-modal datasets allows learning semantically meaningful embeddings in a joint multi-modal representation space without relying on human annotations. These joint embeddings enable zero-shot cross-modal tasks like retrieval and classification. However, these methods often struggle to generalize well on out-of-domain data as they ignore the semantic structure present in modality-specific embeddings. In this context, we propose a novel Semantic-Structure-Preserving Consistency approach to improve generalizability by preserving the modality-specific relationships in the joint embedding space. To capture modality-specific semantic relationships between samples, we propose to learn multiple anchors and represent the multifaceted relationship between samples with respect to their relationship with these anchors. To assign multiple anchors to each sample, we propose a novel Multi-Assignment Sinkhorn-Knopp algorithm. Our experimentation demonstrates that our proposed approach learns semantically meaningful anchors in a self-supervised manner. Furthermore, our evaluation on MSR-VTT and YouCook2 datasets demonstrates that our proposed multi-anchor assignment based solution achieves state-of-the-art performance and generalizes to both inand out-of-domain datasets. Code: https://github.com/Swetha5/Multi_Sinkhorn_Knopp},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023b,

title = {Diffusion Action Segmentation},

author = {Daochang Liu and Qiyue Li and Anh-Dung Dinh and Tingting Jiang and Mubarak Shah and Chang Xu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.17959.pdf

https://arxiv.org/abs/2303.17959

https://finspire13.github.io/DiffAct-Project-Page/

https://github.com/Finspire13/DiffAct

https://youtu.be/o_Jp8shth7U

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Slides.pptx},

doi = { https://doi.org/10.48550/arXiv.2303.17959},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Temporal action segmentation is crucial for understanding long-form videos. Previous works on this task commonly adopt an iterative refinement paradigm by using multi-stage models. We propose a novel framework via denoising diffusion models, which nonetheless shares the same inherent spirit of such iterative refinement. In this framework, action predictions are iteratively generated from random noise with input video features as conditions. To enhance the modeling of three striking characteristics of human actions, including the position prior, the boundary ambiguity, and the relational dependency, we devise a unified masking strategy for the conditioning inputs in our framework. Extensive experiments on three benchmark datasets, i.e., GTEA, 50Salads, and Breakfast, are performed and the proposed method achieves superior or comparable results to state-of-the-art methods, showing the effectiveness of a generative approach for action segmentation.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2023,

title = {TeD-SPAD: Temporal Distinctiveness for Self-supervised Privacy-preservation for video Anomaly Detection},

author = {Joseph Fioresi and Ishan Dave and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.11072.pdf

https://arxiv.org/abs/2308.11072

https://github.com/UCF-CRCV/TeD-SPAD

https://joefioresi718.github.io/TeD-SPAD_webpage/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TeDSPAD_ICCV_poster.pdf

https://youtu.be/3a9qeJUD1GU},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Video-FocalNets: Spatio-Temporal Focal Modulation for Video Action Recognition },

author = {Syed Talal Wasim and Muhammad Uzair Khattak and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.06947.pdf

https://arxiv.org/abs/2307.06947

https://talalwasim.github.io/Video-FocalNets/

https://github.com/TalalWasim/Video-FocalNets

https://talalwasim.github.io/Video-FocalNets/#BibTeX},

doi = { https://doi.org/10.48550/arXiv.2307.06947},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Recent video recognition models utilize Transformer models for long-range spatio-temporal context modeling. Video transformer designs are based on self-attention that can model global context at a high computational cost. In comparison, convolutional designs for videos offer an efficient alternative but lack long-range dependency modeling. Towards achieving the best of both designs, this work proposes Video-FocalNet, an effective and efficient architecture for video recognition that models both local and global contexts. Video-FocalNet is based on a spatio-temporal focal modulation architecture that reverses the interaction and aggregation steps of self-attention for better efficiency. Further, the aggregation step and the interaction step are both implemented using efficient convolution and element-wise multiplication operations that are computationally less expensive than their self-attention counterparts on video representations. We extensively explore the design space of focal modulation-based spatio-temporal context modeling and demonstrate our parallel spatial and temporal encoding design to be the optimal choice. Video-FocalNets perform favorably well against the state-of-the-art transformer-based models for video recognition on three large-scale datasets (Kinetics-400, Kinetics-600, and SS-v2) at a lower computational cost. Our code/models are publicly released.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Efficiently Robustify Pre-Trained Models},

author = {Nishant Jain and Harkirat Behl and Yogesh Singh Rawat and Vibhav Vineet},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV23_Robust_Learning.pdf},

year  = {2023},

date = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {A recent trend in deep learning algorithms has been towards training large scale models, having high parameter count and trained on big dataset. However, robustness of such large scale models towards real-world settings is still a less-explored topic. In this work, we first benchmark the performance of these models under different perturbations and datasets thereby representing real-world shifts, and highlight their degrading performance under these shifts. We then discuss on how complete model fine-tuning based existing robustification schemes might not be a scalable option given very large scale networks and can also lead them to forget some of the desired characteristics. Finally, we propose a simple and cost-effective method to solve this problem, inspired by knowledge transfer literature. It involves robustifying smaller models, at a lower computation cost, and then use them as teachers to tune a fraction of these large scale networks, reducing the overall computational overhead. We evaluate our proposed method under various vision perturbations including ImageNet-C,R,S,A datasets and also for transfer learning, zero-shot evaluation setups on different datasets. Benchmark results show that our method is able to induce robustness to these large scale models efficiently, requiring significantly lower time and also preserves the transfer learning, zero-shot properties of the original model which none of the existing methods are able to achieve. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhou2023,

title = {BT^2 : Backward-compatible Training with Basis Transformation},

author = {Yifei Zhou and Zilu Li and Abhinav Shrivasta and Hengshuang Zhao and Antonio Torralba and Taipeng Tian and Ser-Nam Lim},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2211.03989v3.pdf

https://arxiv.org/abs/2211.03989v3},

doi = {https://doi.org/10.48550/arXiv.2211.03989},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Modern retrieval system often requires recomputing the representation of every piece of data in the gallery when updating to a better representation model. This process is known as backfilling and can be especially costly in the real world where the gallery often contains billions of samples. Recently, researchers have proposed the idea of Backward Compatible Training (BCT) where the new representation model can be trained with an auxiliary loss to make it backward compatible with the old representation. In this way, the new representation can be directly compared with the old representation, in principle avoiding the need for any backfilling. However, followup work shows that there is an inherent tradeoff where a backward compatible representation model cannot simultaneously maintain the performance of the new model itself. This paper reports our ``not-so-surprising'' finding that adding extra dimensions to the representation can help here. However, we also found that naively increasing the dimension of the representation did not work. To deal with this, we propose Backward-compatible Training with a novel Basis Transformation (BT2). A basis transformation (BT) is basically a learnable set of parameters that applies an orthonormal transformation. Such a transformation possesses an important property whereby the original information contained in its input is retained in its output. We show in this paper how a BT can be utilized to add only the necessary amount of additional dimensions. We empirically verify the advantage of BT2 over other state-of-the-art methods in a wide range of settings. We then further extend BT2 to other challenging yet more practical settings, including significant change in model architecture (CNN to Transformers), modality change, and even a series of updates in the model architecture mimicking the evolution of deep learning models.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2023b,

title = {Open-vocabulary Panoptic Segmentation with Embedding Modulation},

author = {Xi Chen and Shuang Li and Ser-Nam Lim and Antonio Torralba and Hengshuang Zhao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.11324.pdf

https://arxiv.org/abs/2303.11324

https://opsnet-page.github.io/},

doi = {https://doi.org/10.48550/arXiv.2303.11324},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Open-vocabulary image segmentation is attracting increasing attention due to its critical applications in the real world. Traditional closed-vocabulary segmentation methods are not able to characterize novel objects, whereas several recent open-vocabulary attempts obtain unsatisfactory results, i.e., notable performance reduction on the closed vocabulary and massive demand for extra data. To this end, we propose OPSNet, an omnipotent and data-efficient framework for Open-vocabulary Panoptic Segmentation. Specifically, the exquisitely designed Embedding Modulation module, together with several meticulous components, enables adequate embedding enhancement and information exchange between the segmentation model and the visual-linguistic well-aligned CLIP encoder, resulting in superior segmentation performance under both open- and closed-vocabulary settings with much fewer need of additional data. Extensive experimental evaluations are conducted across multiple datasets (e.g., COCO, ADE20K, Cityscapes, and PascalContext) under various circumstances, where the proposed OPSNet achieves state-of-the-art results, which demonstrates the effectiveness and generality of the proposed approach. The code and trained models will be made publicly available.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hammoud2023,

title = {Towards a True Evaluation of Rapid Adaptation in Online Continual Learning},

author = {Hasan Abed Al Kader Hammoud and Ameya Prabhu and Ser-Nam Lim and Philip Torr and Adel Bibi and Bernard Ghanem},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2305.09275.pdf

https://arxiv.org/abs/2305.09275

https://github.com/drimpossible/EvalOCL},

doi = {https://doi.org/10.48550/arXiv.2305.09275},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We revisit the common practice of evaluating adaptation of Online Continual Learning (OCL) algorithms through the metric of online accuracy, which measures the accuracy of the model on the immediate next few samples. However, we show that this metric is unreliable, as even vacuous blind classifiers, which do not use input images for prediction, can achieve unrealistically high online accuracy by exploiting spurious label correlations in the data stream. Our study reveals that existing OCL algorithms can also achieve high online accuracy, but perform poorly in retaining useful information, suggesting that they unintentionally learn spurious label correlations. To address this issue, we propose a novel metric for measuring adaptation based on the accuracy on the near-future samples, where spurious correlations are removed. We benchmark existing OCL approaches using our proposed metric on large-scale datasets under various computational budgets and find that better generalization can be achieved by retaining and reusing past seen information. We believe that our proposed metric can aid in the development of truly adaptive OCL methods. We provide code to reproduce our results at https://github.com/drimpossible/EvalOCL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Mitochondria Instance Segmentation with Spatio-Temporal Transformers},

author = {Omkar Thawakar and Rao Muhammad Anwer and Jorma Laaksonen and Orly Reiner and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.12073.pdf

https://github.com/OmkarThawakar/STT-UNET

https://arxiv.org/pdf/2303.12073.pdf

https://link.springer.com/chapter/10.1007/978-3-031-43993-3_59

},

doi = {https://doi.org/10.1007/978-3-031-43993-3_59},

isbn = {978-3-031-43993-3},

year  = {2023},

date = {2023-10-01},

booktitle = {Lecture Notes in Computer Science},

journal = {arXiv:2303.12073},

volume = {14227},

publisher = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},

abstract = {Accurate 3D mitochondria instance segmentation in electron microscopy (EM) is a challenging problem and serves as a prerequisite to empirically analyze their distributions and morphology. Most existing approaches employ 3D convolutions to obtain representative features. However, these convolution-based approaches struggle to effectively capture long-range dependencies in the volume mitochondria data, due to their limited local receptive field. To address this, we propose a hybrid encoder-decoder framework based on a split spatio-temporal attention module that efficiently computes spatial and temporal self-attentions in parallel, which are later fused through a deformable convolution. Further, we introduce a semantic foreground-background adversarial loss during training that aids in delineating the region of mitochondria instances from the background clutter. Our extensive experiments on three benchmarks, Lucchi, MitoEM-R and MitoEM-H, reveal the benefits of the proposed contributions achieving state-of-the-art results on all three datasets. Our code and models are available at https://github.com/OmkarThawakar/STT-UNET.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023,

title = {R2Former: Unified retrieval and ranking Transformer for Place Recognition},

author = {Sijie Zhu and Linjie Yang and Chen Chen and Mubarak Shah and Xiaohui Shen and Heng Wang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR_2023_PlaceRecognitionFinal.pdf

https://arxiv.org/pdf/2304.03410.pdf

https://github.com/Jeff-Zilence/R2Former},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Visual Place Recognition (VPR) estimates the location of query images by matching them with images in a reference database. Conventional methods generally adopt aggregated CNN features for global retrieval and RANSAC-based geometric verification for reranking. However, RANSAC only considers geometric information but ignores other possible information that could be useful for reranking, e.g. local feature correlation, and attention values. In this paper, we propose a unified place recognition framework that handles both retrieval and reranking with a novel transformer model, named R2Former. The proposed reranking module takes feature correlation, attention value, and xy coordinates into account, and learns to determine whether the image pair is from the same location. The whole pipeline is end-to-end trainable and the reranking module alone can

also be adopted on other CNN or transformer backbones as a generic component. Remarkably, R2Former significantly

outperforms state-of-the-art methods on major VPR datasets with much less inference time and memory consumption.

It also achieves the state-of-the-art on the holdout MSLS challenge set and could serve as a simple yet strong solution for real-world large-scale applications. Experiments also show vision transformer tokens are comparable and sometimes better than CNN local features on local matching. The code will be publicly available. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023b,

title = {Class Prototypes based Contrastive Learning for Classifying Multi-Label and Fine-Grained Educational Videos},

author = {Rohit Gupta and Anirban Roy and Sujeong Kim and Claire Christensen and Todd Grindal and Sarah Nixon Gerard and Madeline Cincebeaux and Ajay Divakaran and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Rohit_SRI_CVPR2023_Multi_Modal_Multi_Label_Contrastive_Learning_Camera_Ready-4.pdf

https://www.rohitg.xyz/MMContrast/

https://nusci.csl.sri.com/project/APPROVE},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The recent growth in the consumption of online media by children during early childhood necessitates data-driven tools enabling educators to filter out appropriate educational content for young learners. This paper presents an approach for detecting educational content in online videos. We focus on two widely used educational content classes: literacy and math. For each class, we choose prominent codes (sub-classes) based on the Common Core Standards. For example, literacy codes include ‘letter names’, ‘letter sounds’, and math codes include ‘counting’, ‘sorting’. We pose this as a fine-grained multilabel classification problem as videos can contain multiple types of educational content and the content classes can get visually similar (e.g., ‘letter names’ vs ‘letter sounds’). We propose a novel class prototypes based supervised contrastive learning approach that can handle fine-grained samples associated with multiple labels. We learn a class prototype for each class and a loss function is employed to minimize the distances between a class prototype and the samples from the class. Similarly,

distances between a class prototype and the samples from other classes are maximized. As the alignment between visual

and audio cues are crucial for effective comprehension, we consider a multimodal transformer network to capture the interaction between visual and audio cues in videos while learning the embedding for videos. For evaluation, we present a dataset, APPROVE, employing educational videos from YouTube labeled with fine-grained education classes by education researchers. APPROVE consists of 193 hours of expert-annotated videos with 19 classes. The proposed approach outperforms strong baselines on APPROVE and other benchmarks such as Youtube-8M, and COIN. The dataset is available at https://nusci.csl.sri.com/project/APPROVE.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2023,

title = {TimeBalance: Temporally-Invariant and Temporally-Distinctive Video Representations for Semi-Supervised Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TimeBalance_CVPR23_arxiv.pdf

https://daveishan.github.io/timebalance_webpage/

https://github.com/DAVEISHAN/TimeBalance},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Semi-Supervised Learning can be more beneficial for the video domain compared to images because of its higher annotation

cost and dimensionality. Besides, any video understanding task requires reasoning over both spatial and temporal dimensions. In order to learn both the static and motion related features for the semi-supervised action recognition task, existing methods rely on hard input inductive biases like using two-modalities (RGB and Optical-flow) or two-stream of different playback rates.

Instead of utilizing unlabeled videos through diverse input streams, we rely on self-supervised video representations,

particularly, we utilize temporally-invariant and temporally-distinctive representations. We observe that these representations complement each other depending on the nature of the action. Based on this observation, we propose a student-teacher semi-supervised learning framework, TimeBalance, where we distill the knowledge from a temporally-invariant and a temporally-distinctive teacher. Depending on the nature of the unlabeled video, we dynamically combine the knowledge of these two  teachers based on a novel temporal similarity-based reweighting scheme. Our method achieves state-of-the-art performance

on three action recognition benchmarks: UCF101, HMDB51, and Kinetics400. Code: https://github.com/DAVEISHAN/TimeBalance.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2023,

title = {PivoTAL: Prior-Driven Supervision for Weakly-Supervised Temporal Action Localization},

author = {Mamshad Nayeem Rizve and Gaurav Mittal and Ye Yu and Matthew Hall and Sandra Sajeev and Mubarak Shah and Mei Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023_Supplemental_Material.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR2023_Poster.pdf

https://www.youtube.com/watch?v=6kAoQjXfzio},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Urooj2023,

title = {Learning Situation Hyper-Graphs for Video Question Answering},

author = {Aisha Urooj and Hilde Kuehne and Bo Wu and Kim Chheu and Walid Bousselham and Chuang Gan and Niels Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2023072364-4.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SHG_VQA_CVPR2023_cam_ready_supp.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Answering questions about complex situations in videos requires not only capturing the presence of actors, objects, and their relations but also the evolution of these relationships over time. A situation hyper-graph is a representation that describes situations as scene sub-graphs for video frames and hyper-edges for connected sub-graphs and has been proposed to capture all such information in a compact structured form. In this work, we propose an architecture for Video Question Answering (VQA) that enables answering questions related to video content by predicting situation hyper-graphs, coined Situation Hyper-Graph based Video Question Answering (SHG-VQA). To this end, we train a situation hyper-graph decoder to implicitly identify graph representations with actions and object/human-object relationships from the input video clip. and to use cross-attention

between the predicted situation hyper-graphs and the question embedding to predict the correct answer. The proposed

method is trained in an end-to-end manner and optimized by a VQA loss with the cross-entropy function and a Hungarian

matching loss for the situation graph prediction. The effectiveness of the proposed architecture is extensively evaluated

on two challenging benchmarks: AGQA and STAR. Our results show that learning the underlying situation hypergraphs

helps the system to significantly improve its performance for novel challenges of video question-answering tasks. },

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bhunia2023,

title = {Person Image Synthesis via Denoising Diffusion Model},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Jorma Tapio Laaksonen and Mubarak Shah and Fahad Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/person_image_synthesis_via_den-Camera-ready-PDF.pdf

https://lnkd.in/d-8v3r8B

https://lnkd.in/dGPTjvge

https://lnkd.in/dxcGQsUX

https://github.com/ankanbhunia/PIDM},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The pose-guided person image generation task requires synthesizing photorealistic images of humans in arbitrary poses. The existing approaches use generative adversarial networks that do not necessarily maintain realistic textures or need dense correspondences that struggle to handle complex  deformations and severe occlusions. In this work, we show how denoising diffusion models can be applied for high-fidelity person image synthesis with

strong sample diversity and enhanced mode coverage of the learnt data distribution. Our proposed Person Image Diffusion Model (PIDM) disintegrates the complex transfer problem into a series of simpler forward-backward denoising steps. This helps in learning plausible source-to-target transformation trajectories that result in faithful textures and undistorted appearance details. We introduce a ‘texture diffusion module’ based on cross-attention to accurately model the correspondences between appearance and pose information available in source and target images. Further, we propose  disentangled classifier-free guidance’ to ensure close resemblance between the conditional inputs and the synthesized output in terms of both pose and appearance information. Our extensive results on two large-scale benchmarks and a user study demonstrate the photorealism of our proposed approach under challenging scenarios. We also show how our generated images can help in downstream tasks. Code is available at https://github.com/ankanbhunia/PIDM.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wasim2023,

title = {Vita-CLIP: Video and text adaptive CLIP via Multimodal Prompting},

author = {Syed Talal Wasim and Muzammal Naseer and Salman Khan and Fahad Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/vita_clip_video_and_text_adapt-Camera-ready-PDF.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Adopting contrastive image-text pretrained models like CLIP towards video classification has gained attention due to its cost-effectiveness and competitive performance. However, recent works in this area face a trade-off. Finetuning the pretrained model to achieve strong supervised performance results in low zero-shot generalization. Similarly, freezing the backbone to retain zero-shot capability causes significant drop in supervised accuracy. Because of this,

recent works in literature typically train separate models for supervised and zero-shot action recognition. In this work, we propose a multimodal prompt learning scheme that works to balance the supervised and zero-shot performance under a single unified training. Our prompting approach on the vision side caters for three aspects: 1) Global video-level prompts to model the data distribution; 2) Local frame-level prompts to provide per-frame discriminative

conditioning; and 3) a summary prompt to extract a condensed video representation. Additionally, we define a prompting scheme on the text side to augment the textual context. Through this prompting scheme, we can achieve state-of-the-art zero-shot performance on Kinetics-600, HMDB51 and UCF101 while remaining competitive in the supervised setting. By keeping the pretrained backbone frozen, we optimize a much lower number of parameters and retain the existing general representation which helps achieve the strong zero-shot performance. Our codes and models will be publicly released. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Clark2023,

title = {Where We Are and What We're Looking At: Query Based Worldwide Image Geo-localization Using Hierarchies and Scenes},

author = {Brandon Eric Clark and Alec Kerrigan and Parth Parag Kulkarni and Vicente Vivanco Cepeda and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Camera-Ready-Full-Paper.pdf

https://github.com/AHKerrigan/GeoGuessNet

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR23-Poster_THU-PM-246-1.pdf

https://www.youtube.com/watch?v=fp3hZGbwPqk},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Determining the exact latitude and longitude that a photo was taken is a useful and widely applicable task, yet it remains exceptionally difficult despite the accelerated progress of other computer vision tasks. Most previous approaches have opted to learn single representations of query images, which are then classified at different levels of geographic granularity. These approaches fail to exploit the different visual cues that give context to different hierarchies, such as the country, state, and city level. To this end, we introduce an end-to-end transformer-based architecture that exploits the relationship between different geographic levels (which we refer to as hierarchies) and the corresponding visual scene information in an image through hierarchical cross-attention. We achieve this by learning a query for each geographic hierarchy and scene type. Furthermore, we learn a separate representation for different environmental scenes, as different scenes in the same location are often defined by completely different visual features. We achieve state of the art accuracy on 4 standard geo-localization datasets : Im2GPS, Im2GPS3k, YFCC4k, and YFCC26k, as well as qualitatively demonstrate how our method learns different representations for different visual hierarchies and scenes, which has not been demonstrated in the previous methods. Above previous testing  datasets mostly consist of iconic landmarks or images taken from social media, which makes the dataset a simple memory task, or makes it biased towards certain places. To address this issue we introduce a much harder testing dataset, Google-World-Streets-15k, comprised of images taken from Google Streetview covering the whole planet and present state of the art results. Our code can be found at https://github.com/AHKerrigan/GeoGuessNet.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2023,

title = {Hybrid Active Learning via Deep Clustering for Video Action Detection},

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/hybrid-active-learning-via-deep-clustering-for-video-action-detection/},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chantry2023,

title = {A Large-scale Robustness Analysis of Video Action Recognition Models},

author = {Madeline Chantry and Naman Biyani and Prudvi Kamtam and Shruti Vyas and Hamid Palangi and Vibhav Vineet and Yogesh Rawat},

url = {https://sites.google.com/view/videorobustnessbenchmark/home

https://www.crcv.ucf.edu/research/projects/ucf101-ds-action-recognition-for-real-world-distribution-shifts/

https://github.com/Maddy12/ActionRecognitionRobustnessEval

https://youtu.be/pv2AJ_t-v90

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PosterCVPR2023.png},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023b,

title = {TopNet: Transformer-based Object Placement Network for Image Compositing },

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://arxiv.org/pdf/2304.03372.pdf},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023,

title = {FeatER: An Efficient Network for Human Reconstruction via Feature Map-Based TransformER},

author = {Ce Zheng and Matias Mendieta and Taojiannan Yang and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2205.15448.pdf

https://zczcwh.github.io/feater_page/

https://github.com/zczcwh/FeatER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023b,

title = {POTTER: Pooling Attention Transformer for Efficient Human Mesh Recovery},

author = {Ce Zheng and Xianpeng Liu and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2303.13357.pdf

https://zczcwh.github.io/potter_page/

https://github.com/zczcwh/POTTER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PoseFormerV2: Exploring Frequency Domain for Efficient and Robust 3D Human Pose Estimation},

author = {Qitao Zhao and Ce Zheng and Mengyuan Liu and Pichao Wang and Chen Chen},

url = {https://arxiv.org/pdf/2303.17472.pdf

https://qitaozhao.github.io/PoseFormerV2

https://github.com/QitaoZhao/PoseFormerV2},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2023,

title = {On the Robustness of Large Multimodal Models Against Image Adversarial Attacks},

author = {Xuanming Cui and Alejandro Aparcedo and Young Kyun Jang and Ser-Nam Lim},

url = {https://arxiv.org/pdf/2312.03777

https://arxiv.org/abs/2312.03777},

doi = {https://doi.org/10.48550/arXiv.2312.03777},

year  = {2023},

date = {2023-06-18},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference},

abstract = {Recent advances in instruction tuning have led to the development of State-of-the-Art Large Multimodal Models (LMMs). Given the novelty of these models, the impact of visual adversarial attacks on LMMs has not been thoroughly examined. We conduct a comprehensive study of the robustness of various LMMs against different adversarial attacks, evaluated across tasks including image classification, image captioning, and Visual Question Answer (VQA). We find that in general LMMs are not robust to visual adversarial inputs. However, our findings suggest that context provided to the model via prompts, such as questions in a QA pair helps to mitigate the effects of visual adversarial inputs. Notably, the LMMs evaluated demonstrated remarkable resilience to such attacks on the ScienceQA task with only an 8.10% drop in performance compared to their visual counterparts which dropped 99.73%. We also propose a new approach to real-world image classification which we term query decomposition. By incorporating existence queries into our input prompt we observe diminished attack effectiveness and improvements in image classification accuracy. This research highlights a previously under-explored facet of LMM robustness and sets the stage for future work aimed at strengthening the resilience of multimodal systems in adversarial environments.},

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2023,

title = {3DMODT: Attention-Guided Affinities for Joint Detection & Tracking in 3D Point Clouds},

author = {Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/pdf/2211.00746.pdf},

year  = {2023},

date = {2023-05-29},

urldate = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sangam2023,

title = {TransVisDrone: Spatio-Temporal Transformer for Vision-based Drone-to-Drone Detection in Aerial Videos},

author = {Tushar Sangam and Ishan Rajendrakumar Dave and Waqas Sultani and Mubarak Shah},

url = {https://arxiv.org/pdf/2210.08423.pdf},

year  = {2023},

date = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2023,

title = {AIM: Adapting Image Models for Efficient Video Understanding},

author = {Taojiannan Yang and Yi Zhu and Yusheng Xie and Aston Zhang and Chen Chen and Mu Li},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {AIM: Adapting Image Models for Efficient Video Understanding},

publisher = {Eleventh International Conference on Learning Representations (ICLR)},

abstract = {Recent vision transformer based video models mostly follow the ``image pre-training then finetuning" paradigm and have achieved great success on multiple video benchmarks. However, full finetuning such a video model could be computationally expensive and unnecessary, given the pre-trained image transformer models have demonstrated exceptional transferability. In this work, we propose a novel method to Adapt pre-trained Image Models (AIM) for efficient video understanding. By freezing the pre-trained image model and adding a few lightweight Adapters, we introduce spatial adaptation, temporal adaptation and joint adaptation to gradually equip an image model with spatiotemporal reasoning capability. We show that our proposed AIM can achieve competitive or even better performance than prior arts with substantially fewer tunable parameters on four video action recognition benchmarks. Thanks to its simplicity, our method is also generally applicable to different image pre-trained models, which has the potential to leverage more powerful image foundation models in the future. },

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Re-calibrating Feature Attributions for Model Interpretation},

author = {Peiyu Yang and Naveed Akhtar and Zeyi Wen and Mubarak Shah and Ajmal Mian},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {Re-calibrating Feature Attributions for Model Interpretation},

publisher = {Eleventh International Conference on Learning Representations (ICLR), notable top 25%},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Beetham2023b,

title = {Dual Student Networks for Data-Free Model Stealing},

author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/abs/2309.10058},

year  = {2023},

date = {2023-05-01},

booktitle = {Eleventh International Conference on Learning Representations},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023,

title = {Efficient Distribution Similarity Identification in Clustered Federated Learning via Principal Angles Between Client Data Subspaces},

author = {Saeed Vahidian and Mahdi Morafah and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://arxiv.org/abs/2209.10526},

year  = {2023},

date = {2023-02-07},

urldate = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhong2023,

title = {Refined Semantic Enhancement Towards Frequency Diffusion for Video Captioning},

author = {Xian Zhong and Zipeng Li and Shuqin Chen and Kui Jiang and Chen Chen and Mang Ye},

url = {https://arxiv.org/abs/2211.15076},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023,

title = {Novel Motion Patterns Matter for Practical Skeleton-based Action Recognition},

author = {Mengyuan Liu and Fanyang Meng and Chen Chen and Songtao Wu},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023,

title = {Contrastive Self-Supervised Learning Leads to Higher Adversarial Susceptibility},

author = {Rohit Gupta and Naveed Akhtar and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2207.10862.pdf},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Are all Frames Equal? Active Sparse Labeling for Video Action Detection },

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/active-sparse-labeling-for-video-action-detection/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/neurips_poster_ASL_upload.png

https://github.com/aayushjr/ASL-video },

year  = {2022},

date = {2022-11-28},

urldate = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Video action detection requires annotations at every frame, which drastically increases the labeling cost. In this work, we focus on efficient labeling of videos for action detection to minimize this cost. We propose active sparse labeling (ASL), a novel active learning strategy for video action detection. We propose a novel frame-level scoring mechanism aimed at selecting the most informative frames in a video. We also introduce a novel loss formulation which enables training of action detection model with these sparsely selected frames. We evaluated the proposed approach on two different action detection benchmark datasets, UCF-101-24 and J-HMDB-21, and observed that active sparse labeling can be very effective in saving annotation costs. We demonstrate that the proposed approach performs better than random selection, outperforming all other baselines, with performance comparable to supervised approach using merely 10% annotations.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schiappa2022,

title = {Robustness Analysis of Video-Language Models Against Visual and Language Perturbations},

author = {Madeline Chantry Schiappa and Shruti Vyas and Hamid Palangi and Yogesh Rawat and Vibhav Vineet},

url = {https://sites.google.com/view/videolanguagerobustness/home

https://openreview.net/forum?id=A79jAS4MeW9

https://github.com/Maddy12/VideoLanguageModelRobustness/tree/master},

year  = {2022},

date = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Joint visual and language modeling on large-scale datasets has recently shown good progress in multi-modal tasks when compared to single modal learning. However, robustness of these  approaches against real-world perturbations has not been studied. In this work, we perform the first extensive robustness study of video-language models against various real-world perturbations. We focus on text-to-video retrieval and propose two large-scale benchmark datasets, MSRVTT-P and YouCook2-P, which utilize 90 different visual and 35 different text perturbations. The study reveals some interesting initial findings from the studied models: 1) models are more robust when text is perturbed versus when video is perturbed, 2) models that are pre-trained are more robust than those trained from scratch, 3) models attend more to scene and objects rather than motion and action. We hope this study will serve as a benchmark and guide future research in robust video-language learning. The benchmark introduced in this study along with the code and datasets is available at https://bit.ly/3CNOly4.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xu2022,

title = {Don’t Pour Cereal into Coffee: Differentiable Temporal Logic for Temporal Action Segmentation},

author = {Ziwei Xu and Yogesh Rawat and Yongkang Wong and Mohan Kankanhalli and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ziwei_neurips2022.pdf

https://diff-tl.github.io/

https://github.com/ZiweiXU/DTL-action-segmentation},

year  = {2022},

date = {2022-11-09},

urldate = {2022-11-09},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {We propose Differentiable Temporal Logic (DTL), a model-agnostic framework that introduces temporal constraints to deep networks. DTL treats the outputs of a network as a truth assignment of a temporal logic formula, and computes a temporal logic loss reflecting the consistency between the output and the constraints. We propose a comprehensive set of constraints, which are implicit in data annotations, and incorporate them with deep networks via DTL. We evaluate the effectiveness of DTL on the temporal action segmentation task and observe improved performance and reduced logical errors in the output of different task models. Furthermore, we provide an extensive analysis to visualize the desirable effects of DTL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2022,

title = {GAMa: Cross-view Video Geo-localization},

author = {Shruti Vyas and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512-supp.pdf

https://youtu.be/KSHuer_VXJo},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {The existing work in cross-view geo-localization is based on images where a ground panorama is matched to an aerial image. In this work, we focus on ground videos instead of images which provides ad-ditional contextual cues which are important for this task. There are no existing datasets for this problem, therefore we propose GAMa dataset, a large-scale dataset with ground videos and corresponding aerial images. We also propose a novel approach to solve this problem. At clip-level, a short video clip is matched with corresponding aerial image and is later used to get video-level geo-localization of a long video. Moreover, we propose a hierarchical approach to further improve the clip-level geo-localization. On this challenging dataset, with unaligned images and lim-ited field of view, our proposed method achieves a Top-1 recall rate of 19.4% and 45.1% @1.0mile. Code & dataset are available at this link.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2022,

title = {Med-DANet: Dynamic Architecture Network for Efficient Medical Volumetric Segmentation},

author = {Wenxuan Wang and Chen Chen and Jing Wang and Sen Zha and Yan Zhang and Jiangyun Li},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2206.06575.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {For 3D medical image (e.g. CT and MRI) segmentation, the difficulty of segmenting each slice in a clinical case varies greatly. Previous research on volumetric medical image segmentation in a slice-byslice manner conventionally use the identical 2D deep neural network to segment all the slices of the same case, ignoring the data heterogeneity among image slices. In this paper, we focus on multi-modal 3D MRI brain tumor segmentation and propose a dynamic architecture network named Med-DANet based on adaptive model selection to achieve effective accuracy and efficiency trade-off. For each slice of the input 3D MRI volume, our proposed method learns a slice-specific decision by the Decision Network to dynamically select a suitable model from the predefined Model Bank for the subsequent 2D segmentation task. Extensive experimental results on both BraTS 2019 and 2020 datasets show that our proposed method achieves comparable or better results than previous state-of-the art methods for 3D MRI brain tumor segmentation with much less model complexity. Compared with the state-of-the-art 3D method TransBTS, the proposed framework improves the model efficiency by up to 3.5 × without sacrificing the accuracy. Our code will be publicly available at https://github.com/Wenxuan-1119/Med-DANet.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2022,

title = {GALA: Toward Geometry-and-Lighting-Aware Object Search for Compositing},

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.00125.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GALA_supplementary.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Compositing-aware object search aims to find the most compatible objects for compositing given a background image and a query bounding box. Previous works focus on learning compatibility between the foreground object and background, but fail to learn other important factors from large-scale data, i.e. geometry and lighting. To move a step further, this paper proposes GALA (Geometry-and-Lighting-Aware), a generic foreground object search method with discriminative modeling on geometry and lighting compatibility for open-world image compositing. Remarkably, it achieves state-of-the-art results on the CAIS dataset and generalizes well on large-scale open-world datasets, i.e. Pixabay and

Open Images. In addition, our method can effectively handle non-box scenarios, where users only provide background images without any input bounding box. A web demo (see supplementary materials) is built to showcase applications of the proposed method for compositing-aware search and automatic location/scale prediction for the foreground object. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2022,

title = {Weakly Supervised Grounding for VQA in Vision-Language Transformers},

author = {Aisha Urooj Khan and Hilde Kuehne and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011-supp.pdf

https://github.com/aurooj/WSG-VQA-VLTransformers

https://youtu.be/dekmVb6lq3I},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Transformers for visual-language representation learning have been getting a lot of interest and shown tremendous performance on visual question answering (VQA) and grounding. However, most systems that show good performance of those tasks still rely on pre-trained object

detectors during training, which limits their applicability to the object classes available for those detectors. To mitigate this limitation, this paper

focuses on the problem of weakly supervised grounding in the context of visual question answering in transformers. Our approach leverages

capsules by transforming each visual token into a capsule representation in the visual encoder; it then uses activations from language self-attention layers as a text-guided selection module to mask those capsules before they are forwarded to the next layer. We evaluate our approach on the challenging GQA as well as VQA-HAT dataset for VQA grounding. Our experiments show that: while removing the information of masked

objects from standard transformer architectures leads to a significant drop in performance, the integration of capsules significantly improves the grounding ability of such systems and provides new state-of-the-art results compared to other approaches in the field.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022,

title = {OpenLDN: Learning to Discover Novel Classes for Open-World Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665-supp.pdf

https://github.com/nayeemrizve/OpenLDN

https://youtu.be/p2lYqvklcjA},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Semi-supervised learning (SSL) is one of the dominant approaches to address the annotation bottleneck of supervised learning. Recent SSL methods can effectively leverage a large repository of unlabeled data to improve performance while relying on a small set of labeled data. One common assumption in most SSL methods is that the labeled and unlabeled data are from the same data distribution. However, this is hardly the case in many real-world scenarios, which limits their applicability. In this work, instead, we attempt to solve the challenging open-world SSL problem that does not make such an assumption. In the open-world SSL problem, the objective is to recognize samples of known classes, and simultaneously detect and cluster samples belonging to novel classes present in unlabeled data. This work introduces OpenLDN that utilizes a pairwise similarity loss to discover novel classes. Using a bi-level optimization rule this pairwise similarity loss exploits the information available in the labeled set to implicitly cluster novel class samples, while simultaneously recognizing samples from known classes. After discovering novel  classes, OpenLDN transforms the open-world SSL problem into a standard SSL problem to achieve additional performance gains using existing SSL methods. Our extensive experiments demonstrate that OpenLDN outperforms the current state-of-the-art methods on multiple popular classification benchmarks while providing a better accuracy/training time trade-off. Code: https://github.com/nayeemrizve/OpenLDN},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022b,

title = {Towards Realistic Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402-supp.pdf

https://github.com/nayeemrizve/TRSSL

https://youtu.be/mE7GeQ35WyY},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Deep learning is pushing the state-of-the-art in many computer

vision applications. However, it relies on large annotated data

repositories, and capturing the unconstrained nature of the real-world

data is yet to be solved. Semi-supervised learning (SSL) complements

the annotated training data with a large corpus of unlabeled data to

reduce annotation cost. The standard SSL approach assumes unlabeled

data are from the same distribution as annotated data. Recently, a more

realistic SSL problem, called open-world SSL, is introduced, where the

unannotated data might contain samples from unknown classes. In this paper, we propose a novel pseudo-label based approach to tackle SSL in

open-world setting. At the core of our method, we utilize sample uncertainty and incorporate prior knowledge about class distribution to  generate reliable class-distribution-aware pseudo-labels for unlabeled data belonging to both known and unknown classes. Our extensive experimentation showcases the effectiveness of our approach on several benchmark datasets, where it substantially outperforms the existing state-of-the art on seven diverse datasets including CIFAR-100 (∼17%), ImageNet-100 (∼5%), and Tiny ImageNet (∼9%). We also highlight the flexibility of our approach in solving novel class discovery task, demonstrate its stability in dealing with imbalanced data, and complement our approach with a technique to estimate the number of novel classes. Code: https://github.com/nayeemrizve/TRSSL},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2022,

title = {Self Supervised Learning for Multiple Object Tracking in 3D Point Clouds},

author = {Aakash Kumar and Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/camera_ready_paper.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {2022 IEEE/RSJ International Conference on Intelligent Robots and Systems},

abstract = {Multiple object tracking in 3D point clouds has applications in mobile robots and autonomous driving. This is a challenging problem due to the sparse nature of the point clouds and the added difficulty of annotation in 3D for supervised learning. To overcome these challenges, we propose a neural network architecture that learns effective object features and their affinities in a self supervised fashion for multiple object tracking in 3D point clouds captured with LiDAR sensors. For self supervision, we use two approaches. First, we generate two augmented LiDAR frames from a single real frame by applying translation, rotation and cutout to the objects. Second, we synthesize a LiDAR frame using CAD models or primitive geometric shapes and then apply the above three augmentations to them. Hence, the ground truth object locations and associations are known in both frames for self supervision. This removes the need to annotate object associations in real data, and additionally the need for training data collection and annotation for object detection in synthetic data. To the best of our knowledge, this is the first self supervised multiple object tracking method for 3D data. Our model achieves state of the art results.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Arif2022,

title = {Background-Tolerant Object Classification with Embedded Segmentation Mask for Infrared and Color Imagery},

author = {Maliha Arif and Calvin Yong and Abhijit Mahalanobis and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Final_ICIP2022_MA_submission.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/bg_poster_ICIP2022_Single.jpg},

year  = {2022},

date = {2022-10-16},

urldate = {2022-10-16},

booktitle = {IEEE International Conference on Image Processing},

abstract = {Even though convolutional neural networks (CNNs) can classify objects in images very accurately, it is well known that the attention of the network may not always be on the semantically important regions of the scene. It has been observed that networks often learn background textures, which are not relevant to the object of interest. In turn this makes the networks susceptible to variations and changes in the background

which may negatively affect their performance. 



We propose a new three-step training procedure called split training to reduce this bias in CNNs for object recognition using Infrared imagery and Color (RGB) data. Our split training procedure has three steps. First, a baseline model is trained to recognize objects in images without background, and the activations produced by the higher layers are observed. Next, a second network is trained using Mean Square Error (MSE)  loss to produce the same activations, but in response to the objects embedded in background. This forces the second network to ignore the  background while focusing on the object of interest. Finally, with layers producing the activations frozen, the rest of the second network is  trained using cross-entropy loss to classify the objects in images with background. Our training method outperforms the traditional training procedure in both a simple CNN architecture, as well as for deep CNNs like VGG and DenseNet, and learns to mimic human vision which focuses more on shape and structure than background with higher accuracy. 



Index Terms— infrared imagery, background invariant learning, grad-CAM, split training, MS-COCO},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2023,

title = {DEEPSAR: Vessel Detection In SAR Imagery With Noisy Labels},

author = {Manu S Pillai and Abhijeet Bhattacharya and Tanmay Baweja and Rohit Gupta and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICIP_Submission.pdf},

year  = {2022},

date = {2022-10-16},

urldate = {2023-10-08},

publisher = {IEEE International Conference on Image Processing},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2022b,

title = {Tag-Based Attention Guided Bottom-Up Approach for Video Instance Segmentation},

author = {Jyoti Kini and Mubarak Shah },

url = {https://arxiv.org/pdf/2204.10765.pdf},

year  = {2022},

date = {2022-08-21},

urldate = {2022-08-21},

booktitle = {26th International Conference on Pattern Recognition},

issue = {arxiv:2204.10765},

keywords = {ICPR, Video Instance Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Beetham2022,

title = {Detecting Compromised Architecture/Weights of a Deep Model},

author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},

url = {https://ieeexplore.ieee.org/abstract/document/9956280},

year  = {2022},

date = {2022-08-21},

urldate = {2022-08-21},

booktitle = {26th International Conference on Pattern Recognition},

keywords = {ICPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Self-Supervised Predictive Convolutional Attentive Block for Anomaly Detection},

author = {Nicolae-Catalin Ristea and Neelu Madan and Radu Tudor Ionescu and Kamal Nasrollahi and Fahad Shahbaz Khan and Thomas B. Moeslund and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SSPCAB_camera-arxiv.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Self-Supervised Learning, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UNICON: Combating Label Noise Through Uniform Selection and Contrastive Learning},

author = {Nazmul Karim and Mamshad Nayeem Rizve and Nazanin Rahnavard and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363-supp.pdf

https://github.com/nazmul-karim170/unicon-noisy-label},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Contrastive Learning, CVPR, Noisy Labels, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UBnormal: New Benchmark for Supervised Open-Set Video Anomaly Detection},

author = {Andra Acsintoae and Andrei Florescu and Mariana-Iuliana Georgescu and Tudor Mare and Paul Sumedrea and Radu Tudor Ionescu and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/04315.pdf

https://github.com/lilygeorgescu/UBnormal},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Dataset},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {SPAct: Self-supervised Privacy Preservation for Action Recognition},

author = {Ishan Rajendrakumar Dave and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2203.15205.pdf

https://github.com/DAVEISHAN/SPAct

https://www.youtube.com/watch?v=_PAlMT7ozts},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Action Recognition, CVPR, Privacy Preservation},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {End-to-End Semi-Supervised Learning for Video Action Detection},

author = {Akash Kumar and Yogesh Singh Rawat},

url = {https://arxiv.org/pdf/2203.04251.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Local Learning Matters: Rethinking Data Heterogeneity in Federated Learning},

author = {Matias Mendieta and Taojiannan Yang and Pu Wang and Minwoo Lee and Zhengming Ding and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405_supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Federated Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},

author = {Sijie Zhu and Mubarak Shah and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Cross-View, CVPR, Geo-Localization, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PSTR: End-to-End One-Step Person Search With Transformers},

author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf

https://github.com/JialeCao001/PSTR},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for

discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step

transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging

PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},

keywords = {CVPR, Re-Identification, Transformers, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {OW-DETR: Open-world Detection Transformer},

author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf

https://github.com/akshitac8/OW-DETR.},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while  simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components

namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental

object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},

keywords = {CVPR, Object Detection, Open World, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2022,

title = {Video Action Detection: Analysing Limitations and Challenges},

author = {Rajat Modi and Aayush Jung Rana and Akash Kumar and Praveen Tirupattur and Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.07892.pdf},

year  = {2022},

date = {2022-04-17},

urldate = {2022-04-17},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Performance Evaluation of Boosted 2-stream TCRNet},

author = {Shah Hassan and MdJibanul Haque Jiban and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/paper.pdf},

year  = {2022},

date = {2022-03-02},

urldate = {2022-03-02},

publisher = {International Congress on Information and Communication Technology},

keywords = {ICICT},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pestana2022,

title = {Transferable 3D Adversarial Textures using End-to-end Optimization},

author = {Camilo Pestana and Naveed Akhtar and Nazanin Rahnavard and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/WACV_2022_Deceptive_Textures-1.pdf},

doi = {10.1109/WACV51458.2022.00080},

year  = {2022},

date = {2022-02-15},

urldate = {2022-02-15},

pages = {727-736},

publisher = {IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},

abstract = {Deep visual models are known to be vulnerable to adversarial attacks. The last few years have seen numerous techniques to compute adversarial inputs for these models. However, there are still under-explored avenues in this critical research direction. Among those is the estimation of adversarial textures for 3D models in an end-to-end optimization scheme. In this paper, we propose such a scheme to generate adversarial textures for 3D models that are highly transferable and invariant to different camera views and lighting conditions. Our method makes use of neural rendering with explicit control over the model texture and background. We ensure transferability of the adversarial textures by employing an ensemble of robust and non-robust models. Our technique utilizes 3D models as a proxy to simulate closer to real-life conditions, in contrast to conventional use of 2D images for adversarial attacks. We show the efficacy of our method with extensive experiments.},

keywords = {Adversarial Attacks, WACV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kardan2022,

title = {Self-Joint Supervised Learning},

author = {Navid Kardan and Mitchell Hill and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Self_joint_ICLR-002.pdf

https://github.com/ndkn/Self-joint-Learning},

year  = {2022},

date = {2022-01-20},

urldate = {2022-01-20},

publisher = {International Conference on Learning Representations (ICLR)},

abstract = {Supervised learning is a fundamental framework used to train machine learning systems. A supervised learning problem is often formulated using an i.i.d. as-sumption that restricts model attention to a single relevant signal at a time when predicting. This contrasts with the human ability to actively use related samples as reference when making decisions. We hypothesize that the restriction to a single signal for each prediction in the standard i.i.d. framework contributes to well-known drawbacks of supervised learning: making overconfident predictions and vulnerability to overfitting, adversarial attacks, and out-of-distribution data. To address these limitations, we propose a new supervised learning paradigm called self-joint learning that generalizes the standard approach by modeling the joint conditional distribution of two observed samples, where each sample is an im-age and its label. Rather than assuming samples are independent, our models explicitly learn the sample-to-sample relation of conditional independence. Our framework can naturally incorporate auxiliary unlabeled data to further improve the performance. Experiments on benchmark image datasets show our method offers significant improvement over standard supervised learning in terms of ac-curacy, robustness against adversarial attacks, out-of-distribution detection, and overconfidence mitigation. Code: github.com/ndkn/Self-joint-Learning},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kerrigan2021,

title = {Reformulating Zero-shot Action Recognition for Multi-label Actions},

author = {Alec Kerrigan and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/reformulating_zero_shot_action2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ZSL-Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Reformulating-Zero-shot-Action-Recognition-forMulti-label-Actions.pptx},

year  = {2021},

date = {2021-12-06},

urldate = {2021-12-06},

booktitle = {Thirty-fifth Conference on Neural Information Processing Systems},

keywords = {Action Recognition, NeurIPS, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rajasegaran2021,

title = {Meta-learning the Learning Trends Shared Across Tasks},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.bmvc2021-virtualconference.com/conference/papers/paper_0874.html},

year  = {2021},

date = {2021-11-22},

urldate = {2021-11-22},

booktitle = {British Machine Vision Conference, Nov 22-25},

keywords = {BMVC, Few-Shot Learning, Meta-Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rajasegaran2020,

title = {Self-supervised Knowledge Distillation for Few-shot Learning},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Self-supervised-Knowledge-Distillation-for-Few-shot-Learning.pdf

https://bmvc2021-virtualconference.com/conference/papers/paper_0820.html

https://github.com/brjathu/SKD},

year  = {2021},

date = {2021-11-22},

urldate = {2021-11-22},

booktitle = {British Machine Vision Conference, Nov 22-25},

keywords = {BMVC, Few-Shot Learning, Knowledge Distillation, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Semi-supervised Domain Adaptive Retrieval via Discriminative Hashing Learning},

author = {Haifeng Xia and Taotao Jing and Chen Chen and Zhengming Ding},

url = {https://www.crcv.ucf.edu/chenchen/DHLing_MM_2021.pdf},

doi = {10.1145/3474085.3475526},

year  = {2021},

date = {2021-10-20},

booktitle = {ACM Multimedia (ACM MM), 2021 (Oral)},

keywords = {ACM MM},

pubstate = {published},

tppubtype = {conference}

}

@conference{ZaeemzadehICCV2021,

title = {Face Image Retrieval with Attribute Manipulation},

author = {Alireza Zaeemzadeh and Shabnam Ghadar and Baldo Faieta and Zhe Lin and Nazanin Rahnavard and Mubarak Shah and Ratheesh Kalarot},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/06328.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/06328-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Current face image retrieval solutions are limited, since they treat different facial attributes the same and cannot incorporate

user’s preference for a subset of attributes in their search criteria. This paper introduces a new face image retrieval framework, where the input face query is augmented by both an adjustment vector that specifies the desired modifications

to the facial attributes, and a preference vector that assigns different levels of importance to different attributes. For example, a user can ask for retrieving images similar to a query image, but with a different hair color, and no preference for absence/presence of eyeglasses in the results. To achieve this, we propose to disentangle the semantics, corresponding to various attributes, by learning a set of sparse and orthogonal basis vectors in the latent space of StyleGAN. Such basis vectors are then employed to decompose the dissimilarity between face images in terms of dissimilarity between their attributes, assign preference to the attributes, and adjust the attributes in the query. Enforcing sparsity on the basis vectors helps us to disentangle the latent space and adjust each attribute independently from other attributes, while enforcing orthogonality facilitates preference assignment and the dissimilarity decomposition. The effectiveness of our approach is illustrated by achieving state-of-the-art results for the face image retrieval task. },

keywords = {ICCV, Style GAN, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{RegmiICCV2021,

title = {Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing},

author = {Krishna Regmi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/video_geolocalization_ICCV2021.pdf

https://github.com/kregmi/VTE},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our

GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive

evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at

https://github.com/kregmi/VTE.},

keywords = {Geo-Localization, ICCV, Transformers, Video Geo-localization},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Human Pose Estimation with Spatial and Temporal Transformers},

author = {Ce Zheng and Sijie Zhu and Matias Mendieta and Taojiannan Yang and Chen Chen and Zhengming Ding},

url = {https://arxiv.org/pdf/2103.10455.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

keywords = {Action Recognition, Human Pose Estimation, ICCV, Pose},

pubstate = {published},

tppubtype = {conference}

}

@conference{BhuniaICCV2021,

title = {Handwriting Transformers},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21_supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.



Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.},

keywords = {ICCV, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{NarayanICCV2021,

title = {Discriminative Region-based Multi-Label Zero-Shot Learning},

author = {Sanath Narayan and Akshita Gupta and Salman Khan and Fahad Shahbaz Khan and Ling Shao and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image.  However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues.  We note that the best existing multi-label ZSL method takes a shared approach towards attending  to  region  features  with  a  common  set  of  attention  maps  for  all  the  classes.   Such  shared  maps  lead  to diffused attention, which does not discriminatively focus on relevant  locations  when  the  number  of  classes  are  large. Moreover, mapping spatially-pooled visual features to the class semantics leads to  inter-class  feature entanglement, thus  hampering  the  classification.   Here,  we  propose  an alternate approach towards region-based discriminability-preserving  multi-label  zero-shot  classification. Our  approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information.   The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions,  thereby keeping the multi-class features

disentangled.  Our approach sets a new state of the art on two  large-scale  multi-label  zero-shot  benchmarks:   NUS-WIDE  and  Open  Images.   On  NUS-WIDE,  our  approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results.   Source code is available at <a href="https://github.com/akshitac8/BiAM" target="_blank">https://github.com/akshitac8/BiAM</a>.},

keywords = {ICCV, Transformers, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{ChenICCV2021,

title = {Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos},

author = {Brian Chen and Andrew Rouditchenko and Kevin Duarte and Hilde Kuehne and Samuel Thomas and Angie Boggust and Rameswar Panda and Brian Kingsbury and Rogerio Feris and David Harwatch and James Glass and Michael Picheny and Shih-Fu Chang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.

To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The  resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,

namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets. },

keywords = {ICCV, Multi-Modal Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2021,

title = {Unsupervised Discriminative Embedding for Sub-Action Learning in Complex Activities},

author = {Sirnam Swetha and Hilde Kuehne and Yogesh Singh Rawat and Mubarak Shah },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Unsupervised-Discriminative-Embedding-for-Sub-Action-Learning-in-Complex-Activities.pdf},

year  = {2021},

date = {2021-09-19},

urldate = {2021-09-19},

booktitle = {IEEE International Conference on Image Processing},

keywords = {Action Recognition, ICIP, Un-supervised Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{jiban2021icip,

title = {Two-Stream Boosted TCRNET for Range-Tolerant Infra-Red Target Detection},

author = {Md Jibanul Haque Jiban and Shah Hassan and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Two-Stream_TCRNet__IEEE_ICIP2021.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE Conference on Image Processing},

abstract = {The detection of vehicular targets in infra-red imagery is a challenging task, both due to the relatively few pixels on target and the false alarms produced by the surrounding terrain clutter. It has been previously shown [1] that a relatively simple network (known as TCRNet) can outperform conventional deep CNNs for such applications by maximizing a target to clutter ratio (TCR) metric. In this paper, we introduce a new form of the network (referred to as TCRNet-2) that further improves the performance by first processing target and clutter information in two parallel channels and then combining them to optimize the TCR metric. We also show that the overall performance can be considerably improved by boosting the performance of a primary TCRNet-2 detector, with a secondary

network that enhances discrimination between targets and clutter in the false alarm space of the primary network. We analyze the performance of the proposed networks using a publicly available data set of infra-red images of targets in natural terrain. It is shown that the TCRNet-2 and its boosted version yield considerably better performance than the original TCRNet over a wide range of distances, in both day and night conditions.



Index Terms— TCRNet, Infrared, Target Detection, MWIR, Surveillance},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{ArifICIP2021,

title = {Few Shot Learning for Infra-Red Object Recognition Using Analytically Designed Low Level Filters for Data Representation},

author = {Maliha Arif and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICIP_2021__Sparse_Learning-Camera-Ready.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE International Conference on Image Processing},

abstract = {It is well known that deep convolutional neural networks (CNNs) generalize well over large number of classes when ample training data is available. However, training with smaller datasets does not always achieve robust performance. In such cases, we show that using analytically derived filters in the lowest layer enables a network to achieve better performance than learning from scratch using a relatively small dataset. These class-agnostic filters represent the underlying manifold of the data space, and also generalize to new or unknown classes which may occur on the same manifold. This directly enables new classes to be learned with very few images by simply fine-tuning the final few layers of the network. We illustrate the advantages of our method using the publicly available set of infra-red images of vehicular ground targets. We compare a simple CNN trained using our method with transfer learning performed using the VGG-16 network, and show that when the number of training images is limited, the proposed approach not only achieves better results on the trained classes, but also outperforms a standard network for learning a new object class. 



Index Terms— manifold, eigen representation, few shot learning, sparse learning, infra-red datasets},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{CuellarICIP2021,

title = {Detection of Small Moving Ground Vehicles in Cluttered Terrain Using Infrared Video Imagery},

author = {Adam Cuellar and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/atc_icip.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE International Conference on Image Processing},

abstract = {The detection of small moving targets in cluttered infrared imagery remains a difficult and challenging task. Conventional image subtraction techniques with frame-to-frame registration yield very high false alarm rates. Furthermore, state of the art deep convolutional neural networks (DCNNs) such as YOLO and Mask R-CNN also do not work well for this application. We show however, that it is possible to train a CNN to detect moving targets in a stack of stabilized images

by maximizing a target to clutter ratio (TCR) metric. This metric has been previously used for detecting relatively large stationary targets in single images, but not for the purposes of finding small moving targets using multiple frames. Referred to as moving target indicator network (MTINet), the proposed network does not rely on image subtraction, but instead uses depth-wise convolution to learn inter-frame temporal dependencies. We compare the performance of the MTINet to state of the art DCNNs and a statistical anomaly detection algorithm, and propose a combined approach that offers the benefits of both data-driven learning and statistical analysis. 



Index Terms— Detection, Localization, Infrared, CNN },

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shiraz2021,

title = {Novel View Video Prediction using Dual Representation},

author = {Sarah Shiraz and Krishna Regmi and Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah

},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Projects_Novel-View-Video-Prediction-using-dual-Representation.pdf

https://www.crcv.ucf.edu/research/projects/novel-view-video-prediction-using-dual-representation/},

year  = {2021},

date = {2021-09-19},

urldate = {2021-09-19},

booktitle = {IEEE International Conference on Image Processing},

keywords = {Cross-View, ICIP, View Synthesis},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransBTS: Multimodal Brain Tumor Segmentation Using Transformer},

author = {Wenxuan Wang and Chen Chen and Meng Ding and Jiangyun Li and Hong Yu and Sen Zha},

url = {https://arxiv.org/pdf/2103.04430.pdf},

year  = {2021},

date = {2021-06-26},

booktitle = {International Conference on Medical Image Computing and Computer Assisted Intervention (MICCAI)},

keywords = {MICCAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{georgescu2020anomaly,

title = {Anomaly Detection in Video via Self-Supervised and Multi-Task Learning},

author = {Mariana Iuliana Georgescu and Antonio Bărbălău and Radu Tudor Ionescu and Fahad Shahbaz Khan and Marius Popescu and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.pptx

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Anomaly detection in video is a challenging computer vision problem. Due to the lack of anomalous events at training time, anomaly detection requires the design of learning methods without full supervision. In this paper, we approach anomalous event detection in video through self-supervised and multi-task learning at the object level. We first utilize a pre-trained detector to detect objects. Then, we train a 3D convolutional neural network to produce discriminative anomaly-specific information by jointly learning multiple proxy tasks: three self-supervised and one based on knowledge distillation. The self-supervised tasks are: (i) discrimination of forward/backward moving objects (arrow of time), (ii) discrimination of objects in consecutive/intermittent frames (motion irregularity) and (iii) reconstruction of object-specific appearance information. The knowledge distillation task takes into account both classification and detection information, generating large prediction discrepancies between teacher and student models when anomalies occur. To the best of our knowledge, we are the first to approach anomalous event detection in video as a multi-task learning problem, integrating multiple self-supervised and knowledge distillation proxy tasks in a single architecture. Our lightweight architecture outperforms the state-of-the-art methods on three benchmarks: Avenue, ShanghaiTech and UCSD Ped2. Additionally, we perform an ablation study demonstrating the importance of integrating self-supervised learning and normality-specific distillation in a multi-task learning setting.},

keywords = {Anomaly Detection, Multi-Task Learning, Self-Supervised Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sultani2021,

title = {Dogfight: Detecting Drones from Drones Videos },

author = {Muhammad Waseem Ashraf and Waqas Sultani and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight_Supp.mp4

https://github.com/mwaseema/Drone-Detection

http://im.itu.edu.pk/dogfight-detecting-drones-from-drones-videos/

https://docs.google.com/presentation/d/1huBSbYzyNUCs-gJHdSU2CZ0XfRlBWGOQ/edit#slide=id.p1},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Drone Video Analysis, Object Detection, UAV Video Analysis, UVA Video Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tirupattur2021,

title = {Modeling Multi-Label Action Dependencies for Temporal Action Localization},

author = {Praveen Tirupattur and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization_Supp.zip

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization-CVPR-2021-ORAL.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},

keywords = {CVPR, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zaeemzadeh2021,

title = {Out-of-Distribution Detection Using Union of 1-Dimensional Subspaces},

author = {Alireza Zaeemzadeh and Niccolò Bisagno and Zeno Sambugaro and Nicola Conci and Nazanin Rahnavard and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR21_presentation_video.mov},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Open World, Out of Distribution (OOD)},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2021b,

title = {Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules},

author = {Aisha Urooj Khan and Hilde Kuehne and Kevin Duarte and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr21_poster_v2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr_2021_5min.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {Capsule Networks, CVPR, Grounding, VQA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2021b,

title = {Exploring Complementary Strengths of Invariant and Equivariant Representations for Few-Shot Learning},

author = {Mamshad Nayeem Rizve and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant_Supp.pdf

https://github.com/nayeemrizve/invariance-equivariance

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/FSL_CVPR2021_Video_Final.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Equivariance, Few-Shot Learning, Invariance},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2021,

title = {In Defense of Pseudo-Labeling: An Uncertainty-Aware Pseudo-label Selection Framework for Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/In-Defense-Of-Pseudo-Labeling.pdf

https://github.com/nayeemrizve/ups

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pptx

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_Poster_ICLR2021.png},

year  = {2021},

date = {2021-05-04},

urldate = {2021-05-04},

booktitle = {Ninth International Conference on Learning Representations (ICLR)},

keywords = {ICLR, Network Calibration, Pseudo-Labeling, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2021,

title = {We don’t Need Thousand Proposals: Single Shot Actor-Action Detection in Videos},

author = {Aayush Rana and Yogesh Singh Rawat

},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/12/Projects_Single-shot-actor-action-detection-in-videos.pdf

https://www.crcv.ucf.edu/research/projects/we-dont-need-thousand-proposals-single-shot-actor-action-detection-in-videos/

https://youtu.be/GHKpr5VTbV8},

year  = {2021},

date = {2021-01-05},

urldate = {2021-01-05},

booktitle = {IEEE 2021 Winter Conference on Applications of Computer Vision (WACV)},

keywords = {WACV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2020b,

title = {MMFT-BERT: Multimodal Fusion Transformer with BERT Encodings for Visual Question Answering},

author = {Aisha Urooj Khan and Amir Mazaheri and Niels da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_MMFT-BERT.pdf},

year  = {2020},

date = {2020-11-16},

booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings, EMNLP},

keywords = {EMNLP, Visual Question Answering},

pubstate = {published},

tppubtype = {conference}

}