Publications

@conference{Cepeda2023,

title = {GeoCLIP: Clip-Inspired Alignment between Locations and Images for Effective Worldwide Geo-localization},

author = {Vicente Vivanco Cepeda and Gaurav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_paper.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GeoCLIP_camera_ready_supplementary.pdf

https://vicentevivan.github.io/GeoCLIP/},

year  = {2023},

date = {2023-12-11},

publisher = {Thirty-seventh Conference on Neural Information Processing Systems},

abstract = {Worldwide Geo-localization aims to pinpoint the precise location of images taken anywhere on Earth. This task has considerable challenges due to immense variation in geographic landscapes. The image-to-image retrieval-based approaches fail to solve this problem on a global scale as it is not feasible to construct a large gallery of images covering the entire world. Instead, existing approaches divide the globe into discrete geographic cells, transforming the problem into a classification task. However, their performance is limited by the predefined classes and often results in inaccurate localizations when an image’s location significantly deviates from its class center. To overcome these limitations, we propose GeoCLIP, a novel CLIP-inspired Image-to-GPS retrieval approach that enforces alignment between the image and its corresponding GPS locations. GeoCLIP’s location encoder models the Earth as a continuous function by employing positional encoding through random Fourier features and constructing a hierarchical representation that captures information at varying resolutions to yield a semantically rich highdimensional feature suitable to use even beyond geo-localization. To the best of our knowledge, this is the first work employing GPS encoding for geo-localization. We demonstrate the efficacy of our method via extensive experiments and ablations on benchmark datasets. We achieve competitive performance with just 20% of training data, highlighting its effectiveness even in limited-data settings. Furthermore, we qualitatively demonstrate geo-localization using a text query by leveraging CLIP backbone of our image encoder.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2023,

title = {On Occlusions in Video Action Detection: Benchmark Datasets And Training Recipes},

author = {Rajat Modi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/nips_23.pdf},

year  = {2023},

date = {2023-12-10},

publisher = {NeurIPS 2023.},

abstract = {This paper explores the impact of occlusions in video action detection. We facilitate this study by introducing five new benchmark datasets namely O-UCF and OJHMDB consisting of synthetically controlled static/dynamic occlusions, OVISUCF and OVIS-JHMDB consisting of occlusions with realistic motions and Real-OUCF for occlusions in realistic-world scenarios. We formally confirm an intuitive expectation: existing models suffer a lot as occlusion severity is increased and exhibit different behaviours when occluders are static vs when they are  moving. We discover several curious phenomenon emerging in neural nets: 1) transformers  can naturally outperform CNN models which might have even used occlusion as a form of data augmentation during training 2) incorporating symbolic-components like capsules to such backbones allows them to bind to occluders never even seen during training and 3) Islands of agreement (similar to the ones hypothesized in Hinton et Al’s GLOM) can emerge in realistic images/videos without instance-level supervision, distillation or contrastive-based objectives2(eg. video-textual training). Such emergent properties allow us to derive simple yet effective training recipes which lead to robust occlusion models inductively satisfying the first two stages of the binding mechanism (grouping/segregation). Models leveraging these recipes outperform existing video action-detectors under occlusion by 32.3% on O-UCF, 32.7% on O-JHMDB & 2.6% on Real-OUCF in terms of the vMAP metric.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Frequency Domain Adversarial Training for Robust Volumetric Medical Segmentation},

author = {Asif Hanif and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Frequency-Domain-Adversarial-Training-for-Robust-Volumetric-Medical-Segmentation.pdf

https://github.com/asif-hanif/vafa},

doi = {https://doi.org/10.48550/arXiv.2307.07269},

year  = {2023},

date = {2023-10-08},

publisher = {The 26th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2023},

keywords = {MICCAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2023,

title = {AlignDet: Aligning Pre-training and Fine-tuning in Object Detection},

author = {Ming Li and Jie Wu and Xionghui Wang and Chen Chen and Jie Qin and Xuefeng Xiao and Rui Wang and Min Zheng and Xin Pan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.11077.pdf

https://arxiv.org/abs/2307.11077

https://github.com/liming-ai/AlignDet

https://openreview.net/forum?id=8PA2nX9v_r2

https://liming-ai.github.io/AlignDet/},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2023,

title = {Source-free Domain Adaptive Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.03202.pdf

https://arxiv.org/abs/2308.03202

https://github.com/davidpengucf/SFDAHPE},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Human Pose Estimation (HPE) is widely used in various fields, including motion analysis, healthcare, and virtual reality. However, the great expenses of labeled real-world datasets present a significant challenge for HPE. To  overcome this, one approach is to train HPE models on synthetic datasets and then perform domain adaptation (DA) on realworld data. Unfortunately, existing DA methods for HPE neglect data privacy and security by using both source and target data in the adaptation process. To this end, we propose a new task, named sourcefree domain adaptive HPE, which aims to address the challenges of cross-domain learning of HPE without access to source data during the adaptation process. We further propose a novel framework that consists of three models: source model, intermediate model, and target model, which explores the task from both sourceprotect and target-relevant perspectives. The sourceprotect module preserves source information more effectively while resisting noise, and the target-relevant module reduces the sparsity of spatial representations by building a novel spatial probability space, and pose-specific contrastive learning and information maximization are proposed on the basis of this space. Comprehensive experiments on several domain adaptive HPE benchmarks show that the proposed method outperforms existing approaches by a considerable margin. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {FedPerfix: Towards Partial Model Personalization of Vision Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.09160.pdf

https://arxiv.org/abs/2308.09160

https://github.com/imguangyu/FedPerfix},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We propose and analyze a general framework of federated learning with  partial model personalization. Compared with full model personalization, partial model personalization relies on domain knowledge to select a small portion of the model to personalize, thus imposing a much smaller on-device memory footprint. We propose two federated optimization algorithms for training partially personalized models, where the shared and personal parameters are updated either simultaneously or alternately on each device, but only the shared parameters are communicated and aggregated at the server. We give convergence analyses of both algorithms for minimizing smooth nonconvex functions, providing theoretical support of them for training deep learning models. Our experiments on real-world image and text datasets demonstrate that (a) partial model personalization can obtain most of the benefit of full model personalization with a small fraction of personalized parameters, and, (b) the alternating update algorithm often outperforms the simultaneous update algorithm.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PGFed: Personalize Each Client's Global Objective for Federated Learning},

author = {Jun Luo and Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.01448.pdf

https://github.com/ljaiverson/pgfed},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The mediocre performance of conventional federated learning (FL) over heterogeneous data has been facilitating personalized FL solutions, where, unlike conventional FL which trains a single global consensus model, different models are allowed for different clients. However, in most existing personalized FL algorithms, the collaborative knowledge across the federation was only implicitly passed to the clients in ways such as model aggregation or regularization. We observed that this implicit knowledge transfer fails to maximize the potential value of each client's empirical risk toward other clients. Based on our observation, in this work, we propose Personalized Global Federated Learning (PGFed), a novel personalized FL framework that enables each client to personalize its own global objective by explicitly and adaptively aggregating the empirical risks of itself and other clients. To avoid massive (O(N2)) communication overhead and potential privacy leakage, each client's risk is estimated through a first-order approximation for other clients' adaptive risk aggregation. On top of PGFed, we develop a momentum upgrade, dubbed PGFedMo, to more efficiently utilize clients' empirical risks. Our extensive experiments under different federated settings with benchmark datasets show consistent improvements of PGFed over the compared state-of-the-art alternatives.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {A Large-scale Study of Spatiotemporal Representation Learning with a New Benchmark on Action Recognition},

author = {Andong Dang and Taojiannan Yang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.13505.pdf

https://arxiv.org/abs/2303.13505

https://github.com/AndongDeng/BEAR},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {The goal of building a benchmark (suite of datasets) is to provide a unified protocol for fair evaluation and thus facilitate the evolution of a specific area. Nonetheless, we point out that existing protocols of action recognition could yield partial evaluations due to several limitations. To comprehensively probe the effectiveness of spatiotemporal representation learning, we introduce BEAR, a new BEnchmark on video Action Recognition. BEAR is a collection of 18 video datasets grouped into 5 categories (anomaly, gesture, daily, sports, and instructional), which covers a diverse set of real-world applications. With BEAR, we thoroughly evaluate 6 common spatiotemporal models pre-trained by both supervised and self-supervised learning. We also report transfer performance via standard finetuning, few-shot finetuning, and unsupervised domain adaptation. Our observation suggests that current state-of-the-art cannot solidly guarantee high performance on datasets close to real-world applications, and we hope BEAR can serve as a fair and challenging evaluation benchmark to gain insights on building next-generation spatiotemporal learners. Our dataset, code, and models are released at: https://github.com/AndongDeng/BEAR},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Towards Geospatial Foundation Models via Continual Pretraining},

author = {Matias Mendieta and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2302.04476.pdf

https://arxiv.org/abs/2302.04476

https://github.com/mmendiet/GFM},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Geospatial technologies are becoming increasingly essential in our world for a wide range of applications, including agriculture, urban planning, and disaster response. To help improve the applicability and performance of deep learning models on these geospatial tasks, various works have begun investigating foundation models for this domain. Researchers have explored two prominent approaches for introducing such models in geospatial applications, but both have drawbacks in terms of limited performance benefit or prohibitive training cost. Therefore, in this work, we propose a novel paradigm for building highly effective geospatial foundation models with minimal resource cost and carbon impact. We first construct a compact yet diverse dataset from multiple sources to promote feature diversity, which we term GeoPile. Then, we investigate the potential of continual pretraining from large-scale ImageNet-22k models and propose a multi-objective continual pretraining paradigm, which leverages the strong representations of ImageNet while simultaneously providing the freedom to learn valuable in-domain features. Our approach outperforms previous state-of-the-art geospatial pretraining methods in an extensive evaluation on seven downstream datasets covering various tasks such as change detection, classification, multi-label classification, semantic segmentation, and super-resolution.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Multi-view Self-supervised Disentanglement for General Image Denoising},

author = {Hao Chen and Chenyuan Qu and Yu Zhang and Chen Chen and Jianbo Jiao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Final_Version.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV2023_MeD_Supplymentary_Final_Version.pdf

https://chqwer2.github.io/MeD/

https://github.com/chqwer2/Multi-view-Self-supervised-Disentanglement-Denoising},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {With its significant performance improvements, the deep

learning paradigm has become a standard tool for modern

image denoisers. While promising performance has been

shown on seen noise distributions, existing approaches often suffer from generalisation to unseen noise types or general and real noise. It is understandable as the model is

designed to learn paired mapping (e.g. from a noisy image

to its clean version). In this paper, we instead propose to

learn to disentangle the noisy image, under the intuitive

assumption that different corrupted versions of the same

clean image share a common latent space. A self-supervised

learning framework is proposed to achieve the goal, without looking at the latent clean image. By taking two different corrupted versions of the same image as input, the proposed Multi-view Self-supervised Disentanglement (MeD) approach learns to disentangle the latent clean features from the corruptions and recover the clean image consequently. Extensive experimental analysis on both synthetic and real noise shows the superiority of the proposed method over prior self-supervised approaches, especially on unseen novel noise types. On real noise, the proposed method even outperforms its supervised counterparts by over 3 dB. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {RenderIH: A large-scale synthetic dataset for 3D interacting hand pose estimation},

author = {Lijun Li, Linrui Tian and Xindi Zhang and Qi Wang and Bang Zhang and Liefeng Bo and Mengyuan Liu and Chen Chen},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023b,

title = {When Do Curricula Work in Federated Learning? },

author = {Saeed Vahidian and Sreevatsank Kadaveru and Woonjoon Baek and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2212.12712.pdf

https://arxiv.org/abs/2212.12712},

doi = {https://doi.org/10.48550/arXiv.2212.12712},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {An oft-cited open problem of federated learning is the existence of data heterogeneity at the clients. One pathway to understanding the drastic accuracy drop in federated learning is by scrutinizing the behavior of the clients' deep models on data with different levels of "difficulty", which has been left unaddressed. In this paper, we investigate a different and rarely studied dimension of FL: ordered learning. Specifically, we aim to investigate how ordered learning principles can contribute to alleviating the heterogeneity effects in FL. We present theoretical analysis and conduct extensive empirical studies on the efficacy of orderings spanning three kinds of learning: curriculum, anti-curriculum, and random curriculum. We find that curriculum learning largely alleviates non-IIDness. Interestingly, the more disparate the data distributions across clients the more they benefit from ordered learning. We provide analysis explaining this phenomenon, specifically indicating how curriculum training appears to make the objective landscape progressively less convex, suggesting fast converging iterations at the beginning of the training procedure. We derive quantitative results of convergence for both convex and nonconvex objectives by modeling the curriculum training on federated devices as local SGD with locally biased stochastic gradients. Also, inspired by ordered learning, we propose a novel client selection technique that benefits from the real-world disparity in the clients. Our proposed approach to client selection has a synergic effect when applied together with ordered learning in FL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Samarasinghe2023,

title = {CDFSL-V: Cross-Domain Few-Shot Learning for Videos},

author = {Sarinda Samarasinghe and Mamshad Nayeem and Rizve Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CDFSL_Video_Combined_Final.pdf

https://sarinda251.github.io/CDFSL-V-site/

https://www.youtube.com/watch?v=RdlEzfW013o},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Few-shot video action recognition is an effective approach to recognizing new categories with only a few labeled examples, thereby reducing the challenges associated with collecting and annotating large-scale video datasets. Existing methods in video action recognition rely on large labeled datasets from the same domain. However, this setup is not realistic as novel categories may come from different data domains that may have different spatial and temporal characteristics. This dissimilarity between the source and target domains can pose a significant challenge, rendering traditional few-shot action recognition techniques ineffective. To address this issue, in this work, we propose a novel cross-domain few-shot video action recognition method that leverages self-supervised learning and curriculum learning to balance the information from the source and target domains. To be particular, our method employs a masked autoencoder-based self-supervised training objective to learn from both source and target data in a self-supervised manner. Then a progressive curriculum balances learning the discriminative information from the source dataset with the generic information learned from the target domain. Initially, our curriculum utilizes supervised learning to learn class discriminative features from the source data. As the training progresses, we transition to learning target-domain-specific features. We propose a progressive curriculum to encourage the emergence of rich features in the target domain based on class discriminative supervised features in the source domain. %a schedule that helps with this transition. We evaluate our method on several challenging benchmark datasets and demonstrate that our approach outperforms existing cross-domain few-shot learning techniques.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Preserving Modality Structure Improves Multi-Modal Learning },

author = {Swetha Sirnam and Mamshad Nayeem Rizve and Hilde Kuhne and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.13077.pdf

https://arxiv.org/abs/2308.13077

https://github.com/Swetha5/Multi_Sinkhorn_Knopp

https://swetha5.github.io/MultiSK/

https://youtu.be/1CrGkUATy50

},

doi = {https://doi.org/10.48550/arXiv.2308.13077},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Self-supervised learning on large-scale multi-modal datasets allows learning semantically meaningful embeddings in a joint multi-modal representation space without relying on human annotations. These joint embeddings enable zero-shot cross-modal tasks like retrieval and classification. However, these methods often struggle to generalize well on out-of-domain data as they ignore the semantic structure present in modality-specific embeddings. In this context, we propose a novel Semantic-Structure-Preserving Consistency approach to improve generalizability by preserving the modality-specific relationships in the joint embedding space. To capture modality-specific semantic relationships between samples, we propose to learn multiple anchors and represent the multifaceted relationship between samples with respect to their relationship with these anchors. To assign multiple anchors to each sample, we propose a novel Multi-Assignment Sinkhorn-Knopp algorithm. Our experimentation demonstrates that our proposed approach learns semantically meaningful anchors in a self-supervised manner. Furthermore, our evaluation on MSR-VTT and YouCook2 datasets demonstrates that our proposed multi-anchor assignment based solution achieves state-of-the-art performance and generalizes to both inand out-of-domain datasets. Code: https://github.com/Swetha5/Multi_Sinkhorn_Knopp},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023b,

title = {Diffusion Action Segmentation},

author = {Daochang Liu and Qiyue Li and Anh-Dung Dinh and Tingting Jiang and Mubarak Shah and Chang Xu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.17959.pdf

https://arxiv.org/abs/2303.17959

https://finspire13.github.io/DiffAct-Project-Page/

https://github.com/Finspire13/DiffAct

https://youtu.be/o_Jp8shth7U

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Slides.pptx},

doi = { https://doi.org/10.48550/arXiv.2303.17959},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Temporal action segmentation is crucial for understanding long-form videos. Previous works on this task commonly adopt an iterative refinement paradigm by using multi-stage models. We propose a novel framework via denoising diffusion models, which nonetheless shares the same inherent spirit of such iterative refinement. In this framework, action predictions are iteratively generated from random noise with input video features as conditions. To enhance the modeling of three striking characteristics of human actions, including the position prior, the boundary ambiguity, and the relational dependency, we devise a unified masking strategy for the conditioning inputs in our framework. Extensive experiments on three benchmark datasets, i.e., GTEA, 50Salads, and Breakfast, are performed and the proposed method achieves superior or comparable results to state-of-the-art methods, showing the effectiveness of a generative approach for action segmentation.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2023,

title = {TeD-SPAD: Temporal Distinctiveness for Self-supervised Privacy-preservation for video Anomaly Detection},

author = {Joseph Fioresi and Ishan Dave and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2308.11072.pdf

https://arxiv.org/abs/2308.11072

https://github.com/UCF-CRCV/TeD-SPAD

https://joefioresi718.github.io/TeD-SPAD_webpage/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TeDSPAD_ICCV_poster.pdf

https://youtu.be/3a9qeJUD1GU},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Video-FocalNets: Spatio-Temporal Focal Modulation for Video Action Recognition },

author = {Syed Talal Wasim and Muhammad Uzair Khattak and Muzammal Naseer and Salman Khan and Mubarak Shah and Fahad Shahbaz Khan },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2307.06947.pdf

https://arxiv.org/abs/2307.06947

https://talalwasim.github.io/Video-FocalNets/

https://github.com/TalalWasim/Video-FocalNets

https://talalwasim.github.io/Video-FocalNets/#BibTeX},

doi = { https://doi.org/10.48550/arXiv.2307.06947},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Recent video recognition models utilize Transformer models for long-range spatio-temporal context modeling. Video transformer designs are based on self-attention that can model global context at a high computational cost. In comparison, convolutional designs for videos offer an efficient alternative but lack long-range dependency modeling. Towards achieving the best of both designs, this work proposes Video-FocalNet, an effective and efficient architecture for video recognition that models both local and global contexts. Video-FocalNet is based on a spatio-temporal focal modulation architecture that reverses the interaction and aggregation steps of self-attention for better efficiency. Further, the aggregation step and the interaction step are both implemented using efficient convolution and element-wise multiplication operations that are computationally less expensive than their self-attention counterparts on video representations. We extensively explore the design space of focal modulation-based spatio-temporal context modeling and demonstrate our parallel spatial and temporal encoding design to be the optimal choice. Video-FocalNets perform favorably well against the state-of-the-art transformer-based models for video recognition on three large-scale datasets (Kinetics-400, Kinetics-600, and SS-v2) at a lower computational cost. Our code/models are publicly released.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Efficiently Robustify Pre-Trained Models},

author = {Nishant Jain and Harkirat Behl and Yogesh Singh Rawat and Vibhav Vineet},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICCV23_Robust_Learning.pdf},

year  = {2023},

date = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {A recent trend in deep learning algorithms has been towards training large scale models, having high parameter count and trained on big dataset. However, robustness of such large scale models towards real-world settings is still a less-explored topic. In this work, we first benchmark the performance of these models under different perturbations and datasets thereby representing real-world shifts, and highlight their degrading performance under these shifts. We then discuss on how complete model fine-tuning based existing robustification schemes might not be a scalable option given very large scale networks and can also lead them to forget some of the desired characteristics. Finally, we propose a simple and cost-effective method to solve this problem, inspired by knowledge transfer literature. It involves robustifying smaller models, at a lower computation cost, and then use them as teachers to tune a fraction of these large scale networks, reducing the overall computational overhead. We evaluate our proposed method under various vision perturbations including ImageNet-C,R,S,A datasets and also for transfer learning, zero-shot evaluation setups on different datasets. Benchmark results show that our method is able to induce robustness to these large scale models efficiently, requiring significantly lower time and also preserves the transfer learning, zero-shot properties of the original model which none of the existing methods are able to achieve. },

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhou2023,

title = {BT^2 : Backward-compatible Training with Basis Transformation},

author = {Yifei Zhou and Zilu Li and Abhinav Shrivasta and Hengshuang Zhao and Antonio Torralba and Taipeng Tian and Ser-Nam Lim},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2211.03989v3.pdf

https://arxiv.org/abs/2211.03989v3},

doi = {https://doi.org/10.48550/arXiv.2211.03989},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Modern retrieval system often requires recomputing the representation of every piece of data in the gallery when updating to a better representation model. This process is known as backfilling and can be especially costly in the real world where the gallery often contains billions of samples. Recently, researchers have proposed the idea of Backward Compatible Training (BCT) where the new representation model can be trained with an auxiliary loss to make it backward compatible with the old representation. In this way, the new representation can be directly compared with the old representation, in principle avoiding the need for any backfilling. However, followup work shows that there is an inherent tradeoff where a backward compatible representation model cannot simultaneously maintain the performance of the new model itself. This paper reports our ``not-so-surprising'' finding that adding extra dimensions to the representation can help here. However, we also found that naively increasing the dimension of the representation did not work. To deal with this, we propose Backward-compatible Training with a novel Basis Transformation (BT2). A basis transformation (BT) is basically a learnable set of parameters that applies an orthonormal transformation. Such a transformation possesses an important property whereby the original information contained in its input is retained in its output. We show in this paper how a BT can be utilized to add only the necessary amount of additional dimensions. We empirically verify the advantage of BT2 over other state-of-the-art methods in a wide range of settings. We then further extend BT2 to other challenging yet more practical settings, including significant change in model architecture (CNN to Transformers), modality change, and even a series of updates in the model architecture mimicking the evolution of deep learning models.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2023b,

title = {Open-vocabulary Panoptic Segmentation with Embedding Modulation},

author = {Xi Chen and Shuang Li and Ser-Nam Lim and Antonio Torralba and Hengshuang Zhao},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.11324.pdf

https://arxiv.org/abs/2303.11324

https://opsnet-page.github.io/},

doi = {https://doi.org/10.48550/arXiv.2303.11324},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Open-vocabulary image segmentation is attracting increasing attention due to its critical applications in the real world. Traditional closed-vocabulary segmentation methods are not able to characterize novel objects, whereas several recent open-vocabulary attempts obtain unsatisfactory results, i.e., notable performance reduction on the closed vocabulary and massive demand for extra data. To this end, we propose OPSNet, an omnipotent and data-efficient framework for Open-vocabulary Panoptic Segmentation. Specifically, the exquisitely designed Embedding Modulation module, together with several meticulous components, enables adequate embedding enhancement and information exchange between the segmentation model and the visual-linguistic well-aligned CLIP encoder, resulting in superior segmentation performance under both open- and closed-vocabulary settings with much fewer need of additional data. Extensive experimental evaluations are conducted across multiple datasets (e.g., COCO, ADE20K, Cityscapes, and PascalContext) under various circumstances, where the proposed OPSNet achieves state-of-the-art results, which demonstrates the effectiveness and generality of the proposed approach. The code and trained models will be made publicly available.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hammoud2023,

title = {Towards a True Evaluation of Rapid Adaptation in Online Continual Learning},

author = {Hasan Abed Al Kader Hammoud and Ameya Prabhu and Ser-Nam Lim and Philip Torr and Adel Bibi and Bernard Ghanem},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2305.09275.pdf

https://arxiv.org/abs/2305.09275

https://github.com/drimpossible/EvalOCL},

doi = {https://doi.org/10.48550/arXiv.2305.09275},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {We revisit the common practice of evaluating adaptation of Online Continual Learning (OCL) algorithms through the metric of online accuracy, which measures the accuracy of the model on the immediate next few samples. However, we show that this metric is unreliable, as even vacuous blind classifiers, which do not use input images for prediction, can achieve unrealistically high online accuracy by exploiting spurious label correlations in the data stream. Our study reveals that existing OCL algorithms can also achieve high online accuracy, but perform poorly in retaining useful information, suggesting that they unintentionally learn spurious label correlations. To address this issue, we propose a novel metric for measuring adaptation based on the accuracy on the near-future samples, where spurious correlations are removed. We benchmark existing OCL approaches using our proposed metric on large-scale datasets under various computational budgets and find that better generalization can be achieved by retaining and reusing past seen information. We believe that our proposed metric can aid in the development of truly adaptive OCL methods. We provide code to reproduce our results at https://github.com/drimpossible/EvalOCL.},

keywords = {ICCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Mitochondria Instance Segmentation with Spatio-Temporal Transformers},

author = {Omkar Thawakar and Rao Muhammad Anwer and Jorma Laaksonen and Orly Reiner and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2303.12073.pdf

https://github.com/OmkarThawakar/STT-UNET

https://arxiv.org/pdf/2303.12073.pdf

https://link.springer.com/chapter/10.1007/978-3-031-43993-3_59

},

doi = {https://doi.org/10.1007/978-3-031-43993-3_59},

isbn = {978-3-031-43993-3},

year  = {2023},

date = {2023-10-01},

booktitle = {Lecture Notes in Computer Science},

journal = {arXiv:2303.12073},

volume = {14227},

publisher = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},

abstract = {Accurate 3D mitochondria instance segmentation in electron microscopy (EM) is a challenging problem and serves as a prerequisite to empirically analyze their distributions and morphology. Most existing approaches employ 3D convolutions to obtain representative features. However, these convolution-based approaches struggle to effectively capture long-range dependencies in the volume mitochondria data, due to their limited local receptive field. To address this, we propose a hybrid encoder-decoder framework based on a split spatio-temporal attention module that efficiently computes spatial and temporal self-attentions in parallel, which are later fused through a deformable convolution. Further, we introduce a semantic foreground-background adversarial loss during training that aids in delineating the region of mitochondria instances from the background clutter. Our extensive experiments on three benchmarks, Lucchi, MitoEM-R and MitoEM-H, reveal the benefits of the proposed contributions achieving state-of-the-art results on all three datasets. Our code and models are available at https://github.com/OmkarThawakar/STT-UNET.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023,

title = {R2Former: Unified retrieval and ranking Transformer for Place Recognition},

author = {Sijie Zhu and Linjie Yang and Chen Chen and Mubarak Shah and Xiaohui Shen and Heng Wang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR_2023_PlaceRecognitionFinal.pdf

https://arxiv.org/pdf/2304.03410.pdf

https://github.com/Jeff-Zilence/R2Former},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Visual Place Recognition (VPR) estimates the location of query images by matching them with images in a reference database. Conventional methods generally adopt aggregated CNN features for global retrieval and RANSAC-based geometric verification for reranking. However, RANSAC only considers geometric information but ignores other possible information that could be useful for reranking, e.g. local feature correlation, and attention values. In this paper, we propose a unified place recognition framework that handles both retrieval and reranking with a novel transformer model, named R2Former. The proposed reranking module takes feature correlation, attention value, and xy coordinates into account, and learns to determine whether the image pair is from the same location. The whole pipeline is end-to-end trainable and the reranking module alone can

also be adopted on other CNN or transformer backbones as a generic component. Remarkably, R2Former significantly

outperforms state-of-the-art methods on major VPR datasets with much less inference time and memory consumption.

It also achieves the state-of-the-art on the holdout MSLS challenge set and could serve as a simple yet strong solution for real-world large-scale applications. Experiments also show vision transformer tokens are comparable and sometimes better than CNN local features on local matching. The code will be publicly available. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023b,

title = {Class Prototypes based Contrastive Learning for Classifying Multi-Label and Fine-Grained Educational Videos},

author = {Rohit Gupta and Anirban Roy and Sujeong Kim and Claire Christensen and Todd Grindal and Sarah Nixon Gerard and Madeline Cincebeaux and Ajay Divakaran and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Rohit_SRI_CVPR2023_Multi_Modal_Multi_Label_Contrastive_Learning_Camera_Ready-4.pdf

https://www.rohitg.xyz/MMContrast/

https://nusci.csl.sri.com/project/APPROVE},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The recent growth in the consumption of online media by children during early childhood necessitates data-driven tools enabling educators to filter out appropriate educational content for young learners. This paper presents an approach for detecting educational content in online videos. We focus on two widely used educational content classes: literacy and math. For each class, we choose prominent codes (sub-classes) based on the Common Core Standards. For example, literacy codes include ‘letter names’, ‘letter sounds’, and math codes include ‘counting’, ‘sorting’. We pose this as a fine-grained multilabel classification problem as videos can contain multiple types of educational content and the content classes can get visually similar (e.g., ‘letter names’ vs ‘letter sounds’). We propose a novel class prototypes based supervised contrastive learning approach that can handle fine-grained samples associated with multiple labels. We learn a class prototype for each class and a loss function is employed to minimize the distances between a class prototype and the samples from the class. Similarly,

distances between a class prototype and the samples from other classes are maximized. As the alignment between visual

and audio cues are crucial for effective comprehension, we consider a multimodal transformer network to capture the interaction between visual and audio cues in videos while learning the embedding for videos. For evaluation, we present a dataset, APPROVE, employing educational videos from YouTube labeled with fine-grained education classes by education researchers. APPROVE consists of 193 hours of expert-annotated videos with 19 classes. The proposed approach outperforms strong baselines on APPROVE and other benchmarks such as Youtube-8M, and COIN. The dataset is available at https://nusci.csl.sri.com/project/APPROVE.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2023,

title = {TimeBalance: Temporally-Invariant and Temporally-Distinctive Video Representations for Semi-Supervised Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TimeBalance_CVPR23_arxiv.pdf

https://daveishan.github.io/timebalance_webpage/

https://github.com/DAVEISHAN/TimeBalance},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Semi-Supervised Learning can be more beneficial for the video domain compared to images because of its higher annotation

cost and dimensionality. Besides, any video understanding task requires reasoning over both spatial and temporal dimensions. In order to learn both the static and motion related features for the semi-supervised action recognition task, existing methods rely on hard input inductive biases like using two-modalities (RGB and Optical-flow) or two-stream of different playback rates.

Instead of utilizing unlabeled videos through diverse input streams, we rely on self-supervised video representations,

particularly, we utilize temporally-invariant and temporally-distinctive representations. We observe that these representations complement each other depending on the nature of the action. Based on this observation, we propose a student-teacher semi-supervised learning framework, TimeBalance, where we distill the knowledge from a temporally-invariant and a temporally-distinctive teacher. Depending on the nature of the unlabeled video, we dynamically combine the knowledge of these two  teachers based on a novel temporal similarity-based reweighting scheme. Our method achieves state-of-the-art performance

on three action recognition benchmarks: UCF101, HMDB51, and Kinetics400. Code: https://github.com/DAVEISHAN/TimeBalance.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2023,

title = {PivoTAL: Prior-Driven Supervision for Weakly-Supervised Temporal Action Localization},

author = {Mamshad Nayeem Rizve and Gaurav Mittal and Ye Yu and Matthew Hall and Sandra Sajeev and Mubarak Shah and Mei Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023_Supplemental_Material.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR2023_Poster.pdf

https://www.youtube.com/watch?v=6kAoQjXfzio},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Urooj2023,

title = {Learning Situation Hyper-Graphs for Video Question Answering},

author = {Aisha Urooj and Hilde Kuehne and Bo Wu and Kim Chheu and Walid Bousselham and Chuang Gan and Niels Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2023072364-4.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SHG_VQA_CVPR2023_cam_ready_supp.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Answering questions about complex situations in videos requires not only capturing the presence of actors, objects, and their relations but also the evolution of these relationships over time. A situation hyper-graph is a representation that describes situations as scene sub-graphs for video frames and hyper-edges for connected sub-graphs and has been proposed to capture all such information in a compact structured form. In this work, we propose an architecture for Video Question Answering (VQA) that enables answering questions related to video content by predicting situation hyper-graphs, coined Situation Hyper-Graph based Video Question Answering (SHG-VQA). To this end, we train a situation hyper-graph decoder to implicitly identify graph representations with actions and object/human-object relationships from the input video clip. and to use cross-attention

between the predicted situation hyper-graphs and the question embedding to predict the correct answer. The proposed

method is trained in an end-to-end manner and optimized by a VQA loss with the cross-entropy function and a Hungarian

matching loss for the situation graph prediction. The effectiveness of the proposed architecture is extensively evaluated

on two challenging benchmarks: AGQA and STAR. Our results show that learning the underlying situation hypergraphs

helps the system to significantly improve its performance for novel challenges of video question-answering tasks. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bhunia2023,

title = {Person Image Synthesis via Denoising Diffusion Model},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Jorma Tapio Laaksonen and Mubarak Shah and Fahad Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/person_image_synthesis_via_den-Camera-ready-PDF.pdf

https://lnkd.in/d-8v3r8B

https://lnkd.in/dGPTjvge

https://lnkd.in/dxcGQsUX

https://github.com/ankanbhunia/PIDM},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The pose-guided person image generation task requires synthesizing photorealistic images of humans in arbitrary poses. The existing approaches use generative adversarial networks that do not necessarily maintain realistic textures or need dense correspondences that struggle to handle complex  deformations and severe occlusions. In this work, we show how denoising diffusion models can be applied for high-fidelity person image synthesis with

strong sample diversity and enhanced mode coverage of the learnt data distribution. Our proposed Person Image Diffusion Model (PIDM) disintegrates the complex transfer problem into a series of simpler forward-backward denoising steps. This helps in learning plausible source-to-target transformation trajectories that result in faithful textures and undistorted appearance details. We introduce a ‘texture diffusion module’ based on cross-attention to accurately model the correspondences between appearance and pose information available in source and target images. Further, we propose  disentangled classifier-free guidance’ to ensure close resemblance between the conditional inputs and the synthesized output in terms of both pose and appearance information. Our extensive results on two large-scale benchmarks and a user study demonstrate the photorealism of our proposed approach under challenging scenarios. We also show how our generated images can help in downstream tasks. Code is available at https://github.com/ankanbhunia/PIDM.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wasim2023,

title = {Vita-CLIP: Video and text adaptive CLIP via Multimodal Prompting},

author = {Syed Talal Wasim and Muzammal Naseer and Salman Khan and Fahad Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/vita_clip_video_and_text_adapt-Camera-ready-PDF.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Adopting contrastive image-text pretrained models like CLIP towards video classification has gained attention due to its cost-effectiveness and competitive performance. However, recent works in this area face a trade-off. Finetuning the pretrained model to achieve strong supervised performance results in low zero-shot generalization. Similarly, freezing the backbone to retain zero-shot capability causes significant drop in supervised accuracy. Because of this,

recent works in literature typically train separate models for supervised and zero-shot action recognition. In this work, we propose a multimodal prompt learning scheme that works to balance the supervised and zero-shot performance under a single unified training. Our prompting approach on the vision side caters for three aspects: 1) Global video-level prompts to model the data distribution; 2) Local frame-level prompts to provide per-frame discriminative

conditioning; and 3) a summary prompt to extract a condensed video representation. Additionally, we define a prompting scheme on the text side to augment the textual context. Through this prompting scheme, we can achieve state-of-the-art zero-shot performance on Kinetics-600, HMDB51 and UCF101 while remaining competitive in the supervised setting. By keeping the pretrained backbone frozen, we optimize a much lower number of parameters and retain the existing general representation which helps achieve the strong zero-shot performance. Our codes and models will be publicly released. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Clark2023,

title = {Where We Are and What We're Looking At: Query Based Worldwide Image Geo-localization Using Hierarchies and Scenes},

author = {Brandon Eric Clark and Alec Kerrigan and Parth Parag Kulkarni and Vicente Vivanco Cepeda and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Camera-Ready-Full-Paper.pdf

https://github.com/AHKerrigan/GeoGuessNet

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR23-Poster_THU-PM-246-1.pdf

https://www.youtube.com/watch?v=fp3hZGbwPqk},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Determining the exact latitude and longitude that a photo was taken is a useful and widely applicable task, yet it remains exceptionally difficult despite the accelerated progress of other computer vision tasks. Most previous approaches have opted to learn single representations of query images, which are then classified at different levels of geographic granularity. These approaches fail to exploit the different visual cues that give context to different hierarchies, such as the country, state, and city level. To this end, we introduce an end-to-end transformer-based architecture that exploits the relationship between different geographic levels (which we refer to as hierarchies) and the corresponding visual scene information in an image through hierarchical cross-attention. We achieve this by learning a query for each geographic hierarchy and scene type. Furthermore, we learn a separate representation for different environmental scenes, as different scenes in the same location are often defined by completely different visual features. We achieve state of the art accuracy on 4 standard geo-localization datasets : Im2GPS, Im2GPS3k, YFCC4k, and YFCC26k, as well as qualitatively demonstrate how our method learns different representations for different visual hierarchies and scenes, which has not been demonstrated in the previous methods. Above previous testing  datasets mostly consist of iconic landmarks or images taken from social media, which makes the dataset a simple memory task, or makes it biased towards certain places. To address this issue we introduce a much harder testing dataset, Google-World-Streets-15k, comprised of images taken from Google Streetview covering the whole planet and present state of the art results. Our code can be found at https://github.com/AHKerrigan/GeoGuessNet.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2023,

title = {Hybrid Active Learning via Deep Clustering for Video Action Detection},

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/hybrid-active-learning-via-deep-clustering-for-video-action-detection/},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chantry2023,

title = {A Large-scale Robustness Analysis of Video Action Recognition Models},

author = {Madeline Chantry and Naman Biyani and Prudvi Kamtam and Shruti Vyas and Hamid Palangi and Vibhav Vineet and Yogesh Rawat},

url = {https://sites.google.com/view/videorobustnessbenchmark/home

https://www.crcv.ucf.edu/research/projects/ucf101-ds-action-recognition-for-real-world-distribution-shifts/

https://github.com/Maddy12/ActionRecognitionRobustnessEval

https://youtu.be/pv2AJ_t-v90

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PosterCVPR2023.png},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023b,

title = {TopNet: Transformer-based Object Placement Network for Image Compositing },

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://arxiv.org/pdf/2304.03372.pdf},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023,

title = {FeatER: An Efficient Network for Human Reconstruction via Feature Map-Based TransformER},

author = {Ce Zheng and Matias Mendieta and Taojiannan Yang and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2205.15448.pdf

https://zczcwh.github.io/feater_page/

https://github.com/zczcwh/FeatER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023b,

title = {POTTER: Pooling Attention Transformer for Efficient Human Mesh Recovery},

author = {Ce Zheng and Xianpeng Liu and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2303.13357.pdf

https://zczcwh.github.io/potter_page/

https://github.com/zczcwh/POTTER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PoseFormerV2: Exploring Frequency Domain for Efficient and Robust 3D Human Pose Estimation},

author = {Qitao Zhao and Ce Zheng and Mengyuan Liu and Pichao Wang and Chen Chen},

url = {https://arxiv.org/pdf/2303.17472.pdf

https://qitaozhao.github.io/PoseFormerV2

https://github.com/QitaoZhao/PoseFormerV2},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2023,

title = {3DMODT: Attention-Guided Affinities for Joint Detection & Tracking in 3D Point Clouds},

author = {Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://arxiv.org/pdf/2211.00746.pdf},

year  = {2023},

date = {2023-05-29},

urldate = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sangam2023,

title = {TransVisDrone: Spatio-Temporal Transformer for Vision-based Drone-to-Drone Detection in Aerial Videos},

author = {Tushar Sangam and Ishan Rajendrakumar Dave and Waqas Sultani and Mubarak Shah},

url = {https://arxiv.org/pdf/2210.08423.pdf},

year  = {2023},

date = {2023-05-29},

booktitle = {IEEE International Conference on Robotics and Automation},

keywords = {ICRA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2023,

title = {AIM: Adapting Image Models for Efficient Video Understanding},

author = {Taojiannan Yang and Yi Zhu and Yusheng Xie and Aston Zhang and Chen Chen and Mu Li},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {AIM: Adapting Image Models for Efficient Video Understanding},

publisher = {Eleventh International Conference on Learning Representations (ICLR)},

abstract = {Recent vision transformer based video models mostly follow the ``image pre-training then finetuning" paradigm and have achieved great success on multiple video benchmarks. However, full finetuning such a video model could be computationally expensive and unnecessary, given the pre-trained image transformer models have demonstrated exceptional transferability. In this work, we propose a novel method to Adapt pre-trained Image Models (AIM) for efficient video understanding. By freezing the pre-trained image model and adding a few lightweight Adapters, we introduce spatial adaptation, temporal adaptation and joint adaptation to gradually equip an image model with spatiotemporal reasoning capability. We show that our proposed AIM can achieve competitive or even better performance than prior arts with substantially fewer tunable parameters on four video action recognition benchmarks. Thanks to its simplicity, our method is also generally applicable to different image pre-trained models, which has the potential to leverage more powerful image foundation models in the future. },

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Beetham2023,

title = {Dual Student Networks for Data-Free Model Stealing},

author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {Dual Student Networks for Data-Free Model Stealing },

publisher = {Eleventh International Conference on Learning Representations (ICLR)},

abstract = {Data-free model stealing aims to replicate a target model without direct access to either the training data or the target model. To accomplish this, existing methods use a generator to produce samples in order to train a student model to match the target model outputs. To this end, the two main challenges are estimating gradients of the target model without access to its parameters, and generating a diverse set of images that thoroughly explores the input space. We propose a Dual Student method where two students are symmetrically trained in order to provide the generator a criterion to generate samples that the two students disagree on. On one hand, disagreement on a sample implies at least one student has classified the sample incorrectly when compared with the target model. This push towards disagreeing samples implicitly encourages exploring a more diverse region of input space. On the other hand, our method utilizes gradients of student models to indirectly estimate gradients of the target model. We show that this novel training objective for the generator network is equivalent to optimizing a lower bound on the generator’s loss if we had access to the target model gradients. In other words, our method alters the standard data-free model stealing paradigm by substituting the target model with a separate student model, thereby creating a lower bound which can be directly optimized without additional target model queries or separate synthetic datasets. We show that our new optimization framework provides more accurate gradient estimation of the target model and better accuracies on benchmark classification datasets. Additionally, our approach balances improved query efficiency with training computation cost. Finally, we demonstrate that our method serves as a better proxy model for transfer-based adversarial attacks than existing data-free model stealing methods.},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Re-calibrating Feature Attributions for Model Interpretation},

author = {Peiyu Yang and Naveed Akhtar and Zeyi Wen and Mubarak Shah and Ajmal Mian},

year  = {2023},

date = {2023-05-01},

urldate = {2023-05-01},

booktitle = {Re-calibrating Feature Attributions for Model Interpretation},

publisher = {Eleventh International Conference on Learning Representations (ICLR), notable top 25%},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vahidian2023,

title = {Efficient Distribution Similarity Identification in Clustered Federated Learning via Principal Angles Between Client Data Subspaces},

author = {Saeed Vahidian and Mahdi Morafah and Weijia Wang and Vyacheslav Kungurtsev and Chen Chen and Mubarak Shah and Bill Lin},

url = {https://arxiv.org/abs/2209.10526},

year  = {2023},

date = {2023-02-07},

urldate = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhong2023,

title = {Refined Semantic Enhancement Towards Frequency Diffusion for Video Captioning},

author = {Xian Zhong and Zipeng Li and Shuqin Chen and Kui Jiang and Chen Chen and Mang Ye},

url = {https://arxiv.org/abs/2211.15076},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2023,

title = {Novel Motion Patterns Matter for Practical Skeleton-based Action Recognition},

author = {Mengyuan Liu and Fanyang Meng and Chen Chen and Songtao Wu},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023,

title = {Contrastive Self-Supervised Learning Leads to Higher Adversarial Susceptibility},

author = {Rohit Gupta and Naveed Akhtar and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2207.10862.pdf},

year  = {2023},

date = {2023-02-07},

publisher = {37th AAAI Conference on Artificial Intelligence},

keywords = {AAAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Are all Frames Equal? Active Sparse Labeling for Video Action Detection },

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/active-sparse-labeling-for-video-action-detection/

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/neurips_poster_ASL_upload.png

https://github.com/aayushjr/ASL-video },

year  = {2022},

date = {2022-11-28},

urldate = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Video action detection requires annotations at every frame, which drastically increases the labeling cost. In this work, we focus on efficient labeling of videos for action detection to minimize this cost. We propose active sparse labeling (ASL), a novel active learning strategy for video action detection. We propose a novel frame-level scoring mechanism aimed at selecting the most informative frames in a video. We also introduce a novel loss formulation which enables training of action detection model with these sparsely selected frames. We evaluated the proposed approach on two different action detection benchmark datasets, UCF-101-24 and J-HMDB-21, and observed that active sparse labeling can be very effective in saving annotation costs. We demonstrate that the proposed approach performs better than random selection, outperforming all other baselines, with performance comparable to supervised approach using merely 10% annotations.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schiappa2022,

title = {Robustness Analysis of Video-Language Models Against Visual and Language Perturbations},

author = {Madeline Chantry Schiappa and Shruti Vyas and Hamid Palangi and Yogesh Rawat and Vibhav Vineet},

url = {https://sites.google.com/view/videolanguagerobustness/home

https://openreview.net/forum?id=A79jAS4MeW9

https://github.com/Maddy12/VideoLanguageModelRobustness/tree/master},

year  = {2022},

date = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Joint visual and language modeling on large-scale datasets has recently shown good progress in multi-modal tasks when compared to single modal learning. However, robustness of these  approaches against real-world perturbations has not been studied. In this work, we perform the first extensive robustness study of video-language models against various real-world perturbations. We focus on text-to-video retrieval and propose two large-scale benchmark datasets, MSRVTT-P and YouCook2-P, which utilize 90 different visual and 35 different text perturbations. The study reveals some interesting initial findings from the studied models: 1) models are more robust when text is perturbed versus when video is perturbed, 2) models that are pre-trained are more robust than those trained from scratch, 3) models attend more to scene and objects rather than motion and action. We hope this study will serve as a benchmark and guide future research in robust video-language learning. The benchmark introduced in this study along with the code and datasets is available at https://bit.ly/3CNOly4.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xu2022,

title = {Don’t Pour Cereal into Coffee: Differentiable Temporal Logic for Temporal Action Segmentation},

author = {Ziwei Xu and Yogesh Rawat and Yongkang Wong and Mohan Kankanhalli and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ziwei_neurips2022.pdf

https://diff-tl.github.io/

https://github.com/ZiweiXU/DTL-action-segmentation},

year  = {2022},

date = {2022-11-09},

urldate = {2022-11-09},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {We propose Differentiable Temporal Logic (DTL), a model-agnostic framework that introduces temporal constraints to deep networks. DTL treats the outputs of a network as a truth assignment of a temporal logic formula, and computes a temporal logic loss reflecting the consistency between the output and the constraints. We propose a comprehensive set of constraints, which are implicit in data annotations, and incorporate them with deep networks via DTL. We evaluate the effectiveness of DTL on the temporal action segmentation task and observe improved performance and reduced logical errors in the output of different task models. Furthermore, we provide an extensive analysis to visualize the desirable effects of DTL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2022,

title = {GAMa: Cross-view Video Geo-localization},

author = {Shruti Vyas and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512-supp.pdf

https://youtu.be/KSHuer_VXJo},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {The existing work in cross-view geo-localization is based on images where a ground panorama is matched to an aerial image. In this work, we focus on ground videos instead of images which provides ad-ditional contextual cues which are important for this task. There are no existing datasets for this problem, therefore we propose GAMa dataset, a large-scale dataset with ground videos and corresponding aerial images. We also propose a novel approach to solve this problem. At clip-level, a short video clip is matched with corresponding aerial image and is later used to get video-level geo-localization of a long video. Moreover, we propose a hierarchical approach to further improve the clip-level geo-localization. On this challenging dataset, with unaligned images and lim-ited field of view, our proposed method achieves a Top-1 recall rate of 19.4% and 45.1% @1.0mile. Code & dataset are available at this link.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2022,

title = {Med-DANet: Dynamic Architecture Network for Efficient Medical Volumetric Segmentation},

author = {Wenxuan Wang and Chen Chen and Jing Wang and Sen Zha and Yan Zhang and Jiangyun Li},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2206.06575.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {For 3D medical image (e.g. CT and MRI) segmentation, the difficulty of segmenting each slice in a clinical case varies greatly. Previous research on volumetric medical image segmentation in a slice-byslice manner conventionally use the identical 2D deep neural network to segment all the slices of the same case, ignoring the data heterogeneity among image slices. In this paper, we focus on multi-modal 3D MRI brain tumor segmentation and propose a dynamic architecture network named Med-DANet based on adaptive model selection to achieve effective accuracy and efficiency trade-off. For each slice of the input 3D MRI volume, our proposed method learns a slice-specific decision by the Decision Network to dynamically select a suitable model from the predefined Model Bank for the subsequent 2D segmentation task. Extensive experimental results on both BraTS 2019 and 2020 datasets show that our proposed method achieves comparable or better results than previous state-of-the art methods for 3D MRI brain tumor segmentation with much less model complexity. Compared with the state-of-the-art 3D method TransBTS, the proposed framework improves the model efficiency by up to 3.5 × without sacrificing the accuracy. Our code will be publicly available at https://github.com/Wenxuan-1119/Med-DANet.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2022,

title = {GALA: Toward Geometry-and-Lighting-Aware Object Search for Compositing},

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.00125.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GALA_supplementary.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Compositing-aware object search aims to find the most compatible objects for compositing given a background image and a query bounding box. Previous works focus on learning compatibility between the foreground object and background, but fail to learn other important factors from large-scale data, i.e. geometry and lighting. To move a step further, this paper proposes GALA (Geometry-and-Lighting-Aware), a generic foreground object search method with discriminative modeling on geometry and lighting compatibility for open-world image compositing. Remarkably, it achieves state-of-the-art results on the CAIS dataset and generalizes well on large-scale open-world datasets, i.e. Pixabay and

Open Images. In addition, our method can effectively handle non-box scenarios, where users only provide background images without any input bounding box. A web demo (see supplementary materials) is built to showcase applications of the proposed method for compositing-aware search and automatic location/scale prediction for the foreground object. },

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2022,

title = {Weakly Supervised Grounding for VQA in Vision-Language Transformers},

author = {Aisha Urooj Khan and Hilde Kuehne and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011-supp.pdf

https://github.com/aurooj/WSG-VQA-VLTransformers

https://youtu.be/dekmVb6lq3I},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Transformers for visual-language representation learning have been getting a lot of interest and shown tremendous performance on visual question answering (VQA) and grounding. However, most systems that show good performance of those tasks still rely on pre-trained object

detectors during training, which limits their applicability to the object classes available for those detectors. To mitigate this limitation, this paper

focuses on the problem of weakly supervised grounding in the context of visual question answering in transformers. Our approach leverages

capsules by transforming each visual token into a capsule representation in the visual encoder; it then uses activations from language self-attention layers as a text-guided selection module to mask those capsules before they are forwarded to the next layer. We evaluate our approach on the challenging GQA as well as VQA-HAT dataset for VQA grounding. Our experiments show that: while removing the information of masked

objects from standard transformer architectures leads to a significant drop in performance, the integration of capsules significantly improves the grounding ability of such systems and provides new state-of-the-art results compared to other approaches in the field.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022,

title = {OpenLDN: Learning to Discover Novel Classes for Open-World Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665-supp.pdf

https://github.com/nayeemrizve/OpenLDN

https://youtu.be/p2lYqvklcjA},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Semi-supervised learning (SSL) is one of the dominant approaches to address the annotation bottleneck of supervised learning. Recent SSL methods can effectively leverage a large repository of unlabeled data to improve performance while relying on a small set of labeled data. One common assumption in most SSL methods is that the labeled and unlabeled data are from the same data distribution. However, this is hardly the case in many real-world scenarios, which limits their applicability. In this work, instead, we attempt to solve the challenging open-world SSL problem that does not make such an assumption. In the open-world SSL problem, the objective is to recognize samples of known classes, and simultaneously detect and cluster samples belonging to novel classes present in unlabeled data. This work introduces OpenLDN that utilizes a pairwise similarity loss to discover novel classes. Using a bi-level optimization rule this pairwise similarity loss exploits the information available in the labeled set to implicitly cluster novel class samples, while simultaneously recognizing samples from known classes. After discovering novel  classes, OpenLDN transforms the open-world SSL problem into a standard SSL problem to achieve additional performance gains using existing SSL methods. Our extensive experiments demonstrate that OpenLDN outperforms the current state-of-the-art methods on multiple popular classification benchmarks while providing a better accuracy/training time trade-off. Code: https://github.com/nayeemrizve/OpenLDN},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022b,

title = {Towards Realistic Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402-supp.pdf

https://github.com/nayeemrizve/TRSSL

https://youtu.be/mE7GeQ35WyY},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Deep learning is pushing the state-of-the-art in many computer

vision applications. However, it relies on large annotated data

repositories, and capturing the unconstrained nature of the real-world

data is yet to be solved. Semi-supervised learning (SSL) complements

the annotated training data with a large corpus of unlabeled data to

reduce annotation cost. The standard SSL approach assumes unlabeled

data are from the same distribution as annotated data. Recently, a more

realistic SSL problem, called open-world SSL, is introduced, where the

unannotated data might contain samples from unknown classes. In this paper, we propose a novel pseudo-label based approach to tackle SSL in

open-world setting. At the core of our method, we utilize sample uncertainty and incorporate prior knowledge about class distribution to  generate reliable class-distribution-aware pseudo-labels for unlabeled data belonging to both known and unknown classes. Our extensive experimentation showcases the effectiveness of our approach on several benchmark datasets, where it substantially outperforms the existing state-of-the art on seven diverse datasets including CIFAR-100 (∼17%), ImageNet-100 (∼5%), and Tiny ImageNet (∼9%). We also highlight the flexibility of our approach in solving novel class discovery task, demonstrate its stability in dealing with imbalanced data, and complement our approach with a technique to estimate the number of novel classes. Code: https://github.com/nayeemrizve/TRSSL},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2022,

title = {Self Supervised Learning for Multiple Object Tracking in 3D Point Clouds},

author = {Aakash Kumar and Jyoti Kini and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/camera_ready_paper.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {2022 IEEE/RSJ International Conference on Intelligent Robots and Systems},

abstract = {Multiple object tracking in 3D point clouds has applications in mobile robots and autonomous driving. This is a challenging problem due to the sparse nature of the point clouds and the added difficulty of annotation in 3D for supervised learning. To overcome these challenges, we propose a neural network architecture that learns effective object features and their affinities in a self supervised fashion for multiple object tracking in 3D point clouds captured with LiDAR sensors. For self supervision, we use two approaches. First, we generate two augmented LiDAR frames from a single real frame by applying translation, rotation and cutout to the objects. Second, we synthesize a LiDAR frame using CAD models or primitive geometric shapes and then apply the above three augmentations to them. Hence, the ground truth object locations and associations are known in both frames for self supervision. This removes the need to annotate object associations in real data, and additionally the need for training data collection and annotation for object detection in synthetic data. To the best of our knowledge, this is the first self supervised multiple object tracking method for 3D data. Our model achieves state of the art results.},

keywords = {IROS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Arif2022,

title = {Background-Tolerant Object Classification with Embedded Segmentation Mask for Infrared and Color Imagery},

author = {Maliha Arif and Calvin Yong and Abhijit Mahalanobis and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Final_ICIP2022_MA_submission.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/bg_poster_ICIP2022_Single.jpg},

year  = {2022},

date = {2022-10-16},

urldate = {2022-10-16},

booktitle = {IEEE International Conference on Image Processing},

abstract = {Even though convolutional neural networks (CNNs) can classify objects in images very accurately, it is well known that the attention of the network may not always be on the semantically important regions of the scene. It has been observed that networks often learn background textures, which are not relevant to the object of interest. In turn this makes the networks susceptible to variations and changes in the background

which may negatively affect their performance. 



We propose a new three-step training procedure called split training to reduce this bias in CNNs for object recognition using Infrared imagery and Color (RGB) data. Our split training procedure has three steps. First, a baseline model is trained to recognize objects in images without background, and the activations produced by the higher layers are observed. Next, a second network is trained using Mean Square Error (MSE)  loss to produce the same activations, but in response to the objects embedded in background. This forces the second network to ignore the  background while focusing on the object of interest. Finally, with layers producing the activations frozen, the rest of the second network is  trained using cross-entropy loss to classify the objects in images with background. Our training method outperforms the traditional training procedure in both a simple CNN architecture, as well as for deep CNNs like VGG and DenseNet, and learns to mimic human vision which focuses more on shape and structure than background with higher accuracy. 



Index Terms— infrared imagery, background invariant learning, grad-CAM, split training, MS-COCO},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2023,

title = {DEEPSAR: Vessel Detection In SAR Imagery With Noisy Labels},

author = {Manu S Pillai and Abhijeet Bhattacharya and Tanmay Baweja and Rohit Gupta and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICIP_Submission.pdf},

year  = {2022},

date = {2022-10-16},

urldate = {2023-10-08},

publisher = {IEEE International Conference on Image Processing},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kini2022b,

title = {Tag-Based Attention Guided Bottom-Up Approach for Video Instance Segmentation},

author = {Jyoti Kini and Mubarak Shah },

url = {https://arxiv.org/pdf/2204.10765.pdf},

year  = {2022},

date = {2022-08-21},

urldate = {2022-08-21},

booktitle = {26th International Conference on Pattern Recognition},

issue = {arxiv:2204.10765},

keywords = {ICPR, Video Instance Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Self-Supervised Predictive Convolutional Attentive Block for Anomaly Detection},

author = {Nicolae-Catalin Ristea and Neelu Madan and Radu Tudor Ionescu and Kamal Nasrollahi and Fahad Shahbaz Khan and Thomas B. Moeslund and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SSPCAB_camera-arxiv.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Self-Supervised Learning, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UNICON: Combating Label Noise Through Uniform Selection and Contrastive Learning},

author = {Nazmul Karim and Mamshad Nayeem Rizve and Nazanin Rahnavard and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363-supp.pdf

https://github.com/nazmul-karim170/unicon-noisy-label},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Contrastive Learning, CVPR, Noisy Labels, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UBnormal: New Benchmark for Supervised Open-Set Video Anomaly Detection},

author = {Andra Acsintoae and Andrei Florescu and Mariana-Iuliana Georgescu and Tudor Mare and Paul Sumedrea and Radu Tudor Ionescu and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/04315.pdf

https://github.com/lilygeorgescu/UBnormal},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Dataset},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {SPAct: Self-supervised Privacy Preservation for Action Recognition},

author = {Ishan Rajendrakumar Dave and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2203.15205.pdf

https://github.com/DAVEISHAN/SPAct

https://www.youtube.com/watch?v=_PAlMT7ozts},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Action Recognition, CVPR, Privacy Preservation},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {End-to-End Semi-Supervised Learning for Video Action Detection},

author = {Akash Kumar and Yogesh Singh Rawat},

url = {https://arxiv.org/pdf/2203.04251.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Local Learning Matters: Rethinking Data Heterogeneity in Federated Learning},

author = {Matias Mendieta and Taojiannan Yang and Pu Wang and Minwoo Lee and Zhengming Ding and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405_supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Federated Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},

author = {Sijie Zhu and Mubarak Shah and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Cross-View, CVPR, Geo-Localization, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PSTR: End-to-End One-Step Person Search With Transformers},

author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf

https://github.com/JialeCao001/PSTR},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for

discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step

transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging

PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},

keywords = {CVPR, Re-Identification, Transformers, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {OW-DETR: Open-world Detection Transformer},

author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf

https://github.com/akshitac8/OW-DETR.},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while  simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components

namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental

object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},

keywords = {CVPR, Object Detection, Open World, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2022,

title = {Video Action Detection: Analysing Limitations and Challenges},

author = {Rajat Modi and Aayush Jung Rana and Akash Kumar and Praveen Tirupattur and Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.07892.pdf},

year  = {2022},

date = {2022-04-17},

urldate = {2022-04-17},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Performance Evaluation of Boosted 2-stream TCRNet},

author = {Shah Hassan and MdJibanul Haque Jiban and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/paper.pdf},

year  = {2022},

date = {2022-03-02},

urldate = {2022-03-02},

publisher = {International Congress on Information and Communication Technology},

keywords = {ICICT},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pestana2022,

title = {Transferable 3D Adversarial Textures using End-to-end Optimization},

author = {Camilo Pestana and Naveed Akhtar and Nazanin Rahnavard and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/WACV_2022_Deceptive_Textures-1.pdf},

doi = {10.1109/WACV51458.2022.00080},

year  = {2022},

date = {2022-02-15},

urldate = {2022-02-15},

pages = {727-736},

publisher = {IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},

abstract = {Deep visual models are known to be vulnerable to adversarial attacks. The last few years have seen numerous techniques to compute adversarial inputs for these models. However, there are still under-explored avenues in this critical research direction. Among those is the estimation of adversarial textures for 3D models in an end-to-end optimization scheme. In this paper, we propose such a scheme to generate adversarial textures for 3D models that are highly transferable and invariant to different camera views and lighting conditions. Our method makes use of neural rendering with explicit control over the model texture and background. We ensure transferability of the adversarial textures by employing an ensemble of robust and non-robust models. Our technique utilizes 3D models as a proxy to simulate closer to real-life conditions, in contrast to conventional use of 2D images for adversarial attacks. We show the efficacy of our method with extensive experiments.},

keywords = {Adversarial Attacks, WACV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kardan2022,

title = {Self-Joint Supervised Learning},

author = {Navid Kardan and Mitchell Hill and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Self_joint_ICLR-002.pdf

https://github.com/ndkn/Self-joint-Learning},

year  = {2022},

date = {2022-01-20},

urldate = {2022-01-20},

publisher = {International Conference on Learning Representations (ICLR)},

abstract = {Supervised learning is a fundamental framework used to train machine learning systems. A supervised learning problem is often formulated using an i.i.d. as-sumption that restricts model attention to a single relevant signal at a time when predicting. This contrasts with the human ability to actively use related samples as reference when making decisions. We hypothesize that the restriction to a single signal for each prediction in the standard i.i.d. framework contributes to well-known drawbacks of supervised learning: making overconfident predictions and vulnerability to overfitting, adversarial attacks, and out-of-distribution data. To address these limitations, we propose a new supervised learning paradigm called self-joint learning that generalizes the standard approach by modeling the joint conditional distribution of two observed samples, where each sample is an im-age and its label. Rather than assuming samples are independent, our models explicitly learn the sample-to-sample relation of conditional independence. Our framework can naturally incorporate auxiliary unlabeled data to further improve the performance. Experiments on benchmark image datasets show our method offers significant improvement over standard supervised learning in terms of ac-curacy, robustness against adversarial attacks, out-of-distribution detection, and overconfidence mitigation. Code: github.com/ndkn/Self-joint-Learning},

keywords = {ICLR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kerrigan2021,

title = {Reformulating Zero-shot Action Recognition for Multi-label Actions},

author = {Alec Kerrigan and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/reformulating_zero_shot_action2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ZSL-Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Reformulating-Zero-shot-Action-Recognition-forMulti-label-Actions.pptx},

year  = {2021},

date = {2021-12-06},

urldate = {2021-12-06},

booktitle = {Thirty-fifth Conference on Neural Information Processing Systems},

keywords = {Action Recognition, NeurIPS, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rajasegaran2021,

title = {Meta-learning the Learning Trends Shared Across Tasks},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.bmvc2021-virtualconference.com/conference/papers/paper_0874.html},

year  = {2021},

date = {2021-11-22},

urldate = {2021-11-22},

booktitle = {British Machine Vision Conference, Nov 22-25},

keywords = {BMVC, Few-Shot Learning, Meta-Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rajasegaran2020,

title = {Self-supervised Knowledge Distillation for Few-shot Learning},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Self-supervised-Knowledge-Distillation-for-Few-shot-Learning.pdf

https://bmvc2021-virtualconference.com/conference/papers/paper_0820.html

https://github.com/brjathu/SKD},

year  = {2021},

date = {2021-11-22},

urldate = {2021-11-22},

booktitle = {British Machine Vision Conference, Nov 22-25},

keywords = {BMVC, Few-Shot Learning, Knowledge Distillation, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Semi-supervised Domain Adaptive Retrieval via Discriminative Hashing Learning},

author = {Haifeng Xia and Taotao Jing and Chen Chen and Zhengming Ding},

url = {https://www.crcv.ucf.edu/chenchen/DHLing_MM_2021.pdf},

doi = {10.1145/3474085.3475526},

year  = {2021},

date = {2021-10-20},

booktitle = {ACM Multimedia (ACM MM), 2021 (Oral)},

keywords = {ACM MM},

pubstate = {published},

tppubtype = {conference}

}

@conference{ZaeemzadehICCV2021,

title = {Face Image Retrieval with Attribute Manipulation},

author = {Alireza Zaeemzadeh and Shabnam Ghadar and Baldo Faieta and Zhe Lin and Nazanin Rahnavard and Mubarak Shah and Ratheesh Kalarot},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/06328.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/06328-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Current face image retrieval solutions are limited, since they treat different facial attributes the same and cannot incorporate

user’s preference for a subset of attributes in their search criteria. This paper introduces a new face image retrieval framework, where the input face query is augmented by both an adjustment vector that specifies the desired modifications

to the facial attributes, and a preference vector that assigns different levels of importance to different attributes. For example, a user can ask for retrieving images similar to a query image, but with a different hair color, and no preference for absence/presence of eyeglasses in the results. To achieve this, we propose to disentangle the semantics, corresponding to various attributes, by learning a set of sparse and orthogonal basis vectors in the latent space of StyleGAN. Such basis vectors are then employed to decompose the dissimilarity between face images in terms of dissimilarity between their attributes, assign preference to the attributes, and adjust the attributes in the query. Enforcing sparsity on the basis vectors helps us to disentangle the latent space and adjust each attribute independently from other attributes, while enforcing orthogonality facilitates preference assignment and the dissimilarity decomposition. The effectiveness of our approach is illustrated by achieving state-of-the-art results for the face image retrieval task. },

keywords = {ICCV, Style GAN, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{RegmiICCV2021,

title = {Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing},

author = {Krishna Regmi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/video_geolocalization_ICCV2021.pdf

https://github.com/kregmi/VTE},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our

GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive

evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at

https://github.com/kregmi/VTE.},

keywords = {Geo-Localization, ICCV, Transformers, Video Geo-localization},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {3D Human Pose Estimation with Spatial and Temporal Transformers},

author = {Ce Zheng and Sijie Zhu and Matias Mendieta and Taojiannan Yang and Chen Chen and Zhengming Ding},

url = {https://arxiv.org/pdf/2103.10455.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

keywords = {Action Recognition, Human Pose Estimation, ICCV, Pose},

pubstate = {published},

tppubtype = {conference}

}

@conference{BhuniaICCV2021,

title = {Handwriting Transformers},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21_supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.



Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.},

keywords = {ICCV, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{NarayanICCV2021,

title = {Discriminative Region-based Multi-Label Zero-Shot Learning},

author = {Sanath Narayan and Akshita Gupta and Salman Khan and Fahad Shahbaz Khan and Ling Shao and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image.  However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues.  We note that the best existing multi-label ZSL method takes a shared approach towards attending  to  region  features  with  a  common  set  of  attention  maps  for  all  the  classes.   Such  shared  maps  lead  to diffused attention, which does not discriminatively focus on relevant  locations  when  the  number  of  classes  are  large. Moreover, mapping spatially-pooled visual features to the class semantics leads to  inter-class  feature entanglement, thus  hampering  the  classification.   Here,  we  propose  an alternate approach towards region-based discriminability-preserving  multi-label  zero-shot  classification. Our  approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information.   The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions,  thereby keeping the multi-class features

disentangled.  Our approach sets a new state of the art on two  large-scale  multi-label  zero-shot  benchmarks:   NUS-WIDE  and  Open  Images.   On  NUS-WIDE,  our  approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results.   Source code is available at https://github.com/akshitac8/BiAM.},

keywords = {ICCV, Transformers, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{ChenICCV2021,

title = {Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos},

author = {Brian Chen and Andrew Rouditchenko and Kevin Duarte and Hilde Kuehne and Samuel Thomas and Angie Boggust and Rameswar Panda and Brian Kingsbury and Rogerio Feris and David Harwatch and James Glass and Michael Picheny and Shih-Fu Chang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965-supp.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision},

abstract = {Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.

To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The  resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,

namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets. },

keywords = {ICCV, Multi-Modal Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2021,

title = {Unsupervised Discriminative Embedding for Sub-Action Learning in Complex Activities},

author = {Sirnam Swetha and Hilde Kuehne and Yogesh Singh Rawat and Mubarak Shah },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Unsupervised-Discriminative-Embedding-for-Sub-Action-Learning-in-Complex-Activities.pdf},

year  = {2021},

date = {2021-09-19},

urldate = {2021-09-19},

booktitle = {IEEE International Conference on Image Processing},

keywords = {Action Recognition, ICIP, Un-supervised Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{jiban2021icip,

title = {Two-Stream Boosted TCRNET for Range-Tolerant Infra-Red Target Detection},

author = {Md Jibanul Haque Jiban and Shah Hassan and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Two-Stream_TCRNet__IEEE_ICIP2021.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE Conference on Image Processing},

abstract = {The detection of vehicular targets in infra-red imagery is a challenging task, both due to the relatively few pixels on target and the false alarms produced by the surrounding terrain clutter. It has been previously shown [1] that a relatively simple network (known as TCRNet) can outperform conventional deep CNNs for such applications by maximizing a target to clutter ratio (TCR) metric. In this paper, we introduce a new form of the network (referred to as TCRNet-2) that further improves the performance by first processing target and clutter information in two parallel channels and then combining them to optimize the TCR metric. We also show that the overall performance can be considerably improved by boosting the performance of a primary TCRNet-2 detector, with a secondary

network that enhances discrimination between targets and clutter in the false alarm space of the primary network. We analyze the performance of the proposed networks using a publicly available data set of infra-red images of targets in natural terrain. It is shown that the TCRNet-2 and its boosted version yield considerably better performance than the original TCRNet over a wide range of distances, in both day and night conditions.



Index Terms— TCRNet, Infrared, Target Detection, MWIR, Surveillance},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{ArifICIP2021,

title = {Few Shot Learning for Infra-Red Object Recognition Using Analytically Designed Low Level Filters for Data Representation},

author = {Maliha Arif and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ICIP_2021__Sparse_Learning-Camera-Ready.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE International Conference on Image Processing},

abstract = {It is well known that deep convolutional neural networks (CNNs) generalize well over large number of classes when ample training data is available. However, training with smaller datasets does not always achieve robust performance. In such cases, we show that using analytically derived filters in the lowest layer enables a network to achieve better performance than learning from scratch using a relatively small dataset. These class-agnostic filters represent the underlying manifold of the data space, and also generalize to new or unknown classes which may occur on the same manifold. This directly enables new classes to be learned with very few images by simply fine-tuning the final few layers of the network. We illustrate the advantages of our method using the publicly available set of infra-red images of vehicular ground targets. We compare a simple CNN trained using our method with transfer learning performed using the VGG-16 network, and show that when the number of training images is limited, the proposed approach not only achieves better results on the trained classes, but also outperforms a standard network for learning a new object class. 



Index Terms— manifold, eigen representation, few shot learning, sparse learning, infra-red datasets},

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{CuellarICIP2021,

title = {Detection of Small Moving Ground Vehicles in Cluttered Terrain Using Infrared Video Imagery},

author = {Adam Cuellar and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/atc_icip.pdf},

year  = {2021},

date = {2021-09-19},

publisher = {IEEE International Conference on Image Processing},

abstract = {The detection of small moving targets in cluttered infrared imagery remains a difficult and challenging task. Conventional image subtraction techniques with frame-to-frame registration yield very high false alarm rates. Furthermore, state of the art deep convolutional neural networks (DCNNs) such as YOLO and Mask R-CNN also do not work well for this application. We show however, that it is possible to train a CNN to detect moving targets in a stack of stabilized images

by maximizing a target to clutter ratio (TCR) metric. This metric has been previously used for detecting relatively large stationary targets in single images, but not for the purposes of finding small moving targets using multiple frames. Referred to as moving target indicator network (MTINet), the proposed network does not rely on image subtraction, but instead uses depth-wise convolution to learn inter-frame temporal dependencies. We compare the performance of the MTINet to state of the art DCNNs and a statistical anomaly detection algorithm, and propose a combined approach that offers the benefits of both data-driven learning and statistical analysis. 



Index Terms— Detection, Localization, Infrared, CNN },

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shiraz2021,

title = {Novel View Video Prediction using Dual Representation},

author = {Sarah Shiraz and Krishna Regmi and Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah

},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Projects_Novel-View-Video-Prediction-using-dual-Representation.pdf

https://www.crcv.ucf.edu/research/projects/novel-view-video-prediction-using-dual-representation/},

year  = {2021},

date = {2021-09-19},

urldate = {2021-09-19},

booktitle = {IEEE International Conference on Image Processing},

keywords = {Cross-View, ICIP, View Synthesis},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransBTS: Multimodal Brain Tumor Segmentation Using Transformer},

author = {Wenxuan Wang and Chen Chen and Meng Ding and Jiangyun Li and Hong Yu and Sen Zha},

url = {https://arxiv.org/pdf/2103.04430.pdf},

year  = {2021},

date = {2021-06-26},

booktitle = {International Conference on Medical Image Computing and Computer Assisted Intervention (MICCAI)},

keywords = {MICCAI},

pubstate = {published},

tppubtype = {conference}

}

@conference{georgescu2020anomaly,

title = {Anomaly Detection in Video via Self-Supervised and Multi-Task Learning},

author = {Mariana Iuliana Georgescu and Antonio Bărbălău and Radu Tudor Ionescu and Fahad Shahbaz Khan and Marius Popescu and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.pptx

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Anomaly-Detection-in-Video-via-Self-Supervised-and-Multi-Task-Learning.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Anomaly detection in video is a challenging computer vision problem. Due to the lack of anomalous events at training time, anomaly detection requires the design of learning methods without full supervision. In this paper, we approach anomalous event detection in video through self-supervised and multi-task learning at the object level. We first utilize a pre-trained detector to detect objects. Then, we train a 3D convolutional neural network to produce discriminative anomaly-specific information by jointly learning multiple proxy tasks: three self-supervised and one based on knowledge distillation. The self-supervised tasks are: (i) discrimination of forward/backward moving objects (arrow of time), (ii) discrimination of objects in consecutive/intermittent frames (motion irregularity) and (iii) reconstruction of object-specific appearance information. The knowledge distillation task takes into account both classification and detection information, generating large prediction discrepancies between teacher and student models when anomalies occur. To the best of our knowledge, we are the first to approach anomalous event detection in video as a multi-task learning problem, integrating multiple self-supervised and knowledge distillation proxy tasks in a single architecture. Our lightweight architecture outperforms the state-of-the-art methods on three benchmarks: Avenue, ShanghaiTech and UCSD Ped2. Additionally, we perform an ablation study demonstrating the importance of integrating self-supervised learning and normality-specific distillation in a multi-task learning setting.},

keywords = {Anomaly Detection, Multi-Task Learning, Self-Supervised Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sultani2021,

title = {Dogfight: Detecting Drones from Drones Videos },

author = {Muhammad Waseem Ashraf and Waqas Sultani and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight_Supp.mp4

https://github.com/mwaseema/Drone-Detection

http://im.itu.edu.pk/dogfight-detecting-drones-from-drones-videos/

https://docs.google.com/presentation/d/1huBSbYzyNUCs-gJHdSU2CZ0XfRlBWGOQ/edit#slide=id.p1},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Drone Video Analysis, Object Detection, UAV Video Analysis, UVA Video Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tirupattur2021,

title = {Modeling Multi-Label Action Dependencies for Temporal Action Localization},

author = {Praveen Tirupattur and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization_Supp.zip

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization-CVPR-2021-ORAL.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},

keywords = {CVPR, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zaeemzadeh2021,

title = {Out-of-Distribution Detection Using Union of 1-Dimensional Subspaces},

author = {Alireza Zaeemzadeh and Niccolò Bisagno and Zeno Sambugaro and Nicola Conci and Nazanin Rahnavard and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR21_presentation_video.mov},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Open World, Out of Distribution (OOD)},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2021b,

title = {Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules},

author = {Aisha Urooj Khan and Hilde Kuehne and Kevin Duarte and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr21_poster_v2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr_2021_5min.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {Capsule Networks, CVPR, Grounding, VQA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2021b,

title = {Exploring Complementary Strengths of Invariant and Equivariant Representations for Few-Shot Learning},

author = {Mamshad Nayeem Rizve and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant_Supp.pdf

https://github.com/nayeemrizve/invariance-equivariance

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/FSL_CVPR2021_Video_Final.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Equivariance, Few-Shot Learning, Invariance},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2021,

title = {In Defense of Pseudo-Labeling: An Uncertainty-Aware Pseudo-label Selection Framework for Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah },

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/In-Defense-Of-Pseudo-Labeling.pdf

https://github.com/nayeemrizve/ups

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pptx

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_Poster_ICLR2021.png},

year  = {2021},

date = {2021-05-04},

urldate = {2021-05-04},

booktitle = {Ninth International Conference on Learning Representations (ICLR)},

keywords = {ICLR, Network Calibration, Pseudo-Labeling, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2021,

title = {We don’t Need Thousand Proposals: Single Shot Actor-Action Detection in Videos},

author = {Aayush Rana and Yogesh Singh Rawat

},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/12/Projects_Single-shot-actor-action-detection-in-videos.pdf

https://www.crcv.ucf.edu/research/projects/we-dont-need-thousand-proposals-single-shot-actor-action-detection-in-videos/

https://youtu.be/GHKpr5VTbV8},

year  = {2021},

date = {2021-01-05},

urldate = {2021-01-05},

booktitle = {IEEE 2021 Winter Conference on Applications of Computer Vision (WACV)},

keywords = {WACV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2020b,

title = {MMFT-BERT: Multimodal Fusion Transformer with BERT Encodings for Visual Question Answering},

author = {Aisha Urooj Khan and Amir Mazaheri and Niels da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_MMFT-BERT.pdf},

year  = {2020},

date = {2020-11-16},

booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings, EMNLP},

keywords = {EMNLP, Visual Question Answering},

pubstate = {published},

tppubtype = {conference}

}

@conference{McIntosh2020,

title = {Target Detection in Cluttered Environments Using Infra-Red Images},

author = {Bruce McIntosh and Shashanka Venkataramanan and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/icip2020_photoready.pdf},

year  = {2020},

date = {2020-10-28},

booktitle = {IEEE International Conference on Image Processing},

abstract = {The detection of targets in infra-red imagery is a challenging problem which involves locating small targets in heavily

cluttered environments while maintaining a low false alarm rate. We propose a network that optimizes a “target to clutter

ratio”(TCR) metric defined as the ratio of the output energies produced by the network in response to targets and clutter.

We show that for target detection, it is advantageous to analytically derive the first layer of a CNN to maximize the

TCR metric, and then train the rest of the network to optimize the same cost function. We evaluate the performance of the

resulting network using a public domain MWIR data set released by the US Army’s Night Vision Laboratories, and compare

it to the state-of-the-art detectors such as Faster RCNN and Yolo-v3. Referred to as the TCRNet, the proposed network

demonstrates state of the art results with greater than 30% improvement in probability of detection while reducing

the false alarm rate by more than a factor of 2 when compared to these leading methods. Ablation studies also show that the

proposed approach and metric are superior to learning the entire network from scratch, or using conventional regression

metrics such as the mean square error (MSE). },

keywords = {ICIP},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2020,

title = {Simultaneous Detection and Tracking with Motion Modelling for Multiple Object Tracking},

author = {ShiJie Sun and Naveed Akhtar and XiangYu Song and HuanSheng Song and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking_Supp.pdf

https://shijies.github.io/DMMN_Page/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Deep learning based Multiple Object Tracking (MOT) currently relies on off-the-shelf detectors for tracking-by-detection. This results in deep models that are detector biased and evaluations that are detector influenced. To resolve this issue, we introduce Deep Motion Modeling Network (DMM-Net) that can estimate multiple objects’ motion parameters to perform joint detection and association in an end-to-end manner. DMM-Net models object features over multiple frames and simultaneously infers object classes, visibility and their motion parameters. These outputs are readily used to update the tracklets for efficient MOT. DMM-Net achieves PR-MOTA score of 12.80 @ 120+ fps for the popular UA-DETRAC challenge - which is better performance and orders of magnitude faster. We also contribute a synthetic large-scale public dataset Omni-MOT for vehicle tracking that provides precise ground-truth annotations to eliminate the detector influence in MOT evaluation. This 14M+ frames dataset is extendable with our public script (Code at Dataset, Dataset Recorder, Omni-MOT Source). We demonstrate the suitability of Omni-MOT for deep learning with DMM-Net, and also make the source code of our network public.},

keywords = {ECCV, Tracking},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xie2020,

title = {Count- and Similarity-aware R-CNN for Pedestrian Detection},

author = {Jin Xie and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Yanwei Pang and Ling Shao and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Count-and-Similarity-aware-R-CNN-for-Pedestrian-Detection.pdf},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Recent pedestrian detection methods generally rely on additional supervision, such as visible bounding-box annotations, to handle heavy occlusions. We propose an approach that leverages pedestrian count and proposal similarity information within a two-stage pedestrian detection framework. Both pedestrian count and proposal similarity are derived from standard full-body annotations commonly used to train pedestrian detectors. We introduce a count-weighted detection loss function that assigns higher weights to the detection errors occurring at highly overlapping pedestrians. The proposed loss function is utilized at both stages of the two-stage detector. We further introduce a count-andsimilarity branch within the two-stage detection framework, which predicts pedestrian count as well as proposal similarity to identify distinct proposals. Our approach requires neither part information nor visible bounding-box annotations. Experiments are performed on the CityPersons and CrowdHuman datasets. Our method sets a new state-of-the-art on both datasets. Further, it achieves an absolute gain of 2.4% over the current state-of-the-art, in terms of log-average miss rate, on the heavily occluded (HO) set of CityPersons test set, without using additional visible bounding-box supervision. Finally, we demonstrate the applicability of our approach for the problem of human instance segmentation. Code and models are available at: https://github.com/Leotju/CaSe.},

keywords = {Detection, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schatz2020,

title = {A Recurrent Transformer Network for Novel View Action Synthesis},

author = {Kara Marie Schatz and Erik Quintanilla and Shruti Vyas and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_A-Recurrent-Transformer-Network-for-Novel-View-Action-Synthesis.pdf

https://www.crcv.ucf.edu/research/projects/a-recurrent-transformer-network-for-novel-view-action-synthesis/},

year  = {2020},

date = {2020-08-23},

urldate = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of synthesizing human actions from novel views. Given an input video of an actor performing some action, we aim to synthesize a video with the same action performed from a novel view with the help of an appearance prior. We propose an end-to-end deep network to solve this problem. The proposed network utilizes the change in viewpoint to transform the action from the input view to the novel view in feature space. The transformed action is integrated with the target appearance using the proposed  recurrent transformer network, which provides a transformed appearance for each time-step in the action sequence. The recurrent transformer network utilize action key-points which are determined in an unsupervised approach using the encoded action features. We also propose a hierarchical structure for the recurrent transformation which further improves the performance. We demonstrate the effectiveness of the proposed method through extensive experiments conducted on a large-scale multi-view action recognition NTU-RGB+D dataset. In addition, we show that the proposed method can transform the action to a novel viewpoint with an entirely different scene or actor. The code is publicly available at https://github.com/schatzkara/cross-view-video.},

keywords = {Cross-View, ECCV, REU, View Action Synthesis, View Synthesis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2020,

title = {Multi-view Action Recognition using Cross-view Video Prediction},

author = {Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction_Supp.pdf

https://www.crcv.ucf.edu/research/projects/multi-view-action-recognition-using-cross-view-video-prediction/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of action recognition in a multi-view environment. Most of the existing approaches utilize  pose information for multi-view action recognition. We focus on RGB modality instead and propose an unsupervised representation learning framework, which encodes the scene dynamics in videos captured from multiple viewpoints via predicting actions from unseen views. The framework takes multiple short video clips from different viewpoints and time as input and learns an holistic internal representation which is used to predict a video clip from an unseen viewpoint and time. The ability of the proposed network to render unseen video frames enables it to learn a meaningful and robust representation of the scene dynamics. We evaluate the effectiveness of the learned representation for multiview video action recognition in a supervised approach. We observe a significant improvement in the performance with RGB modality on NTU-RGB+D dataset, which is the largest dataset for multi-view action recognition. The proposed framework also achieves state-of-the-art results with depth modality, which validates the generalization capability of the approach to other data modalities. The code is publicly available at https://github.com/svyas23/cross-view-action.},

keywords = {Cross-View Video Prediction, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Venkataramanan2020,

title = {Attention Guided Anomaly Localization in Images},

author = {Shashanka Venkataramanan and Kuan-Chuan Peng and Rajat Vikram Singh and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/2813.pdf

https://youtu.be/b-EQr-fGPWo},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Anomaly localization is an important problem in computer vision which involves localizing anomalous regions within images with applications in industrial inspection, surveillance, and medical imaging. This task is challenging due to the small sample size and pixel coverage of the anomaly in real-world scenarios. Most prior works need to use anomalous training images to compute a class-specific threshold to localize anomalies. Without the need of anomalous training images, we propose Convolutional Adversarial Variational autoencoder with Guided Attention (CAVGA), which localizes the anomaly with a convolutional latent variable to preserve the spatial information. In the unsupervised setting, we propose an attention expansion loss where we encourage CAVGA to focus on all normal regions in the image. Furthermore, in the weakly supervised setting we propose a complementary guided attention loss, where we encourage the attention map to focus on all normal regions while minimizing the attention map corresponding to anomalous regions in the image. CAVGA outperforms the state-of-the-art (SOTA) anomaly localization methods on MVTec Anomaly Detection (MVTAD), modified ShanghaiTech Campus (mSTC) and Large-scale Attention based Glaucoma (LAG) datasets in the unsupervised setting and when using only 2% anomalous images in the weakly-supervised setting. CAVGA also outperforms SOTA anomaly detection methods on the MNIST, CIFAR-10, Fashion-MNIST, MVTAD, mSTC and LAG datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jonediei2020,

title = {Select to Better Learn: Fast and Accurate Deep Learning using Data Selection from Nonlinear Manifolds},

author = {Mohsen Joneidi and Saeed Vahidian and Ashkan Esmaeili and Weijia Wang and Nazanin Rahnavard and Bill Lin and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/04/Select-to-Better-Learn.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/04/Select-to-Better-Learn_Supplementary.pdf},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Finding a small subset of data whose linear combination spans other data points, also called column subset selection problem (CSSP), is an important open problem in computer science with many applications in computer vision and deep learning such as the ones shown in Fig. 1. There are some studies that solve CSSP in a polynomial time complexity w.r.t. the size of the original dataset. A simple and efficient selection algorithm with a linear complexity order, referred to as spectrum pursuit (SP), is proposed that pursuits spectral components of the dataset using available sample points. The proposed non-greedy algorithm aims to iteratively find K data samples whose span is close to that of the first K spectral components of entire data. SP has no parameter to be fine tuned and this desirable property makes it problem-independent. The simplicity of SP enables us to extend the underlying linear model to more complex models such as nonlinear manifolds and graph-based models. The nonlinear extension of SP is introduced as kernel-SP (KSP). The superiority of the proposed algorithms is demonstrated in a wide range of applications.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Duarte2020,

title = {Visual-textual Capsule Routing for Text-based Video Segmentation},

author = {Bruce McIntosh and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation_Supplementary.zip

https://www.crcv.ucf.edu/research/projects/visual-textual-capsule-routing-for-text-based-video-segmentation/},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},

edition = {(Oral)},

abstract = {Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization. },

keywords = {Capsule Networks, CVPR, Video Object Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2020,

title = {iTAML : An Incremental Task-Agnostic Meta-learning Approach},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/04/iTAML.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/04/iTAML-Supplementary.pdf},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Humans can continuously learn new knowledge as their experience grows. In contrast, previous learning in deep neural networks can quickly fade out when they are trained on a new task. In this paper, we hypothesize this problem can be avoided by learning a set of generalized parameters, that are neither specific to old nor new tasks. In this pursuit, we introduce a novel meta-learning approach that seeks to maintain an equilibrium between all the encountered tasks. This is ensured by a new meta-update rule which avoids catastrophic forgetting. In comparison to previous metalearning techniques, our approach is task-agnostic. When presented with a continuum of data, our model automatically identifies the task and quickly adapts to it with just a single update. We perform extensive experiments on five datasets in a class-incremental setting, leading to significant improvements over the state of the art methods (e.g., a 21.3% boost on CIFAR100 with 10 incremental tasks). Specifically, on large-scale datasets that generally prove difficult cases for incremental learning, our approach delivers absolute gains as high as 19.1% and 7.4% on ImageNet and MS-Celeb datasets, respectively. Our codes are available at: https://github.com/brjathu/iTAML. },

keywords = {CVPR, Meta-Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sharghi2020,

title = {Text Synopsis Generation for Egocentric Videos},

author = {Aidean Sharghi and Niels da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/05/Publications_Text-Synopsis-Generation-for-Egocentric-Videos.pdf

https://youtu.be/Z2Rpy7MC7QI},

year  = {2020},

date = {2020-05-08},

booktitle = {International Conference on Pattern Recognition},

keywords = {Egocentric, ICPR, Text Synopsis, Video Summarization},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2020,

title = {Gabriella: An Online System for Real-Time Activity Detection in Untrimmed Security Videos},

author = {Mamshad Nayeem Rizve and Ugur Demir and Praveen Tirupattur and Aayush Rana and Kevin Duarte and Ishan Dave and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/05/Gabriella.pdf

https://www.crcv.ucf.edu/research/projects/gabriella-an-online-system-for-real-time-activity-detection-in-untrimmed-security-videos/},

year  = {2020},

date = {2020-04-23},

urldate = {2020-04-23},

booktitle = {25th International Conference on Pattern Recognition, Italy, 10-15 January 2021 (ICPR 2020)},

keywords = {Activity Detection, IARPA DIVA, ICPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2020,

title = {RescueNet: Joint Building Segmentation and Damage Assessment from Satellite Imagery},

author = {Rohit Gupta and Mubarak Shah},

url = {https://arxiv.org/pdf/2004.07312.pdf},

year  = {2020},

date = {2020-04-15},

booktitle = {International Conference on Pattern Recognition},

keywords = {Geo-Localization, ICPR, Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Edraki2019,

title = {Subspace Capsule Network},

author = {Marzieh Edraki and Nazanin Rahnavard and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/03/Projects_SubspaceCapsuleNetwork.pdf

https://www.crcv.ucf.edu/research/projects/subspace-capsule-network/},

year  = {2020},

date = {2020-02-07},

booktitle = {34th Conference on Association for the Advancement of Artificial Intelligence (AAAI 2020), New York, USA},

keywords = {Capsule Networks},

pubstate = {published},

tppubtype = {conference}

}

@conference{LaLonde2019,

title = {INN: Inflated Neural Networks for IPMN Diagnosis},

author = {Rodney LaLonde and Irene Tanner and Nikiforaki, K. and Georgios Z. Papadakis and Kandel, P. and Bolan, CW and M.B. Wallace and Ulas Bagci},

url = {https://doi.org/10.1007/978-3-030-32254-0_12},

year  = {2019},

date = {2019-12-20},

booktitle = {MICCAI},

keywords = {Medical},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khosravan2019,

title = {PAN: Projective Adversarial Network for Medical Image Segmentation},

author = {Naji Khosravan and Aliasghar Mortazi and M.B. Wallace and Ulas Bagci

},

url = {https://doi.org/10.1007/978-3-030-32226-7_8},

year  = {2019},

date = {2019-12-19},

booktitle = {MICCAI},

keywords = {Medical},

pubstate = {published},

tppubtype = {conference}

}

@conference{Mortazi2019,

title = {Weakly Supervised Segmentation by A Deep Geodesic Prior},

author = {Aliasghar Mortazi and Naji Khosravan and Torigian, DA and Kurugol, S. and Ulas Bagci

},

url = {https://doi.org/10.1007/978-3-030-32692-0_28},

year  = {2019},

date = {2019-12-17},

booktitle = {MICCAI 2019-MLMI},

keywords = {Medical},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2019,

title = {Cross-modality Knowledge Transfer for Prostate Segmentation from CT Scans},

author = {Liu, Y. and Naji Khosravan and Liu, Y. and Stember, J. and Ulas Bagci and S. Jambawalikar},

url = {https://doi.org/10.1007/978-3-030-33391-1_8},

year  = {2019},

date = {2019-12-15},

booktitle = {MICCAI 2019-DART},

keywords = {Medical},

pubstate = {published},

tppubtype = {conference}

}

@conference{Anwar2019,

title = {A Survey on Recent Advancements for AI-Enabled Radiomics in Neuro-Oncology},

author = {Anwar, S. and Tooba, A. and Rafique, K. and RaviPrakash, H. and Mohy-ud-din, H. and Ulas Bagci},

url = {https://doi.org/10.1007/978-3-030-40124-5_3},

year  = {2019},

date = {2019-12-10},

booktitle = {MICCAI 2019-RNO-AI},

keywords = {Medical},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khodadadeh2019,

title = {Unsupervised Meta-Learning for Few-Shot Image Classification},

author = {Siavash Khodadadeh and Ladislau Bölöni and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/01/Publications_unsupervised-meta-learning-for-few-shot-image-classification.pdf

https://www.crcv.ucf.edu/research/projects/unsupervised-meta-learning-for-few-shot-image-and-video-classification/},

year  = {2019},

date = {2019-11-30},

booktitle = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

journal = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

keywords = {Meta-Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sultani2019,

title = {Human Action Recognition in Drone Videos using a Few Aerial Training Examples},

author = {Waqas Sultani and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2019/10/1910.10027v1.pdf},

year  = {2019},

date = {2019-11-04},

publisher = {Cornell University Library, arXiv:1910.10027v1. [cs.CV]},

keywords = {Action Recognition},

pubstate = {published},

tppubtype = {conference}

}

@conference{Duarte2019,

title = {CapsuleVOS: Semi-Supervised Video Object Segmentation Using Capsule Routing},

author = {Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_CapsuleVOS-Semi-Supervised-Video-Object-Segmentation-Using-Capsule-Routing.pdf

https://www.crcv.ucf.edu/research/projects/capsulevos-semi-supervised-video-object-segmentation-using-capsule-routing/},

year  = {2019},

date = {2019-10-30},

publisher = { International Conference on Computer Vision (ICCV 2019), Seoul, South Korea, Oct 27-Nov 2},

keywords = {Video Object Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Alemu2019,

title = {Deep Constrained Dominant Sets for Person Re-Identification},

author = {Leulseged Tesfaye Alemu and Marcello Pelillo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2019/08/Publications_Deep-Constrained-Dominant-Sets-for-Person-Re-Identification.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2019/08/Publications_Deep-Constrained-Dominant-Sets-for-Person-Re-Identification_Supplementary.pdf

https://www.crcv.ucf.edu/research/projects/deep-constrained-dominant-sets-for-person-re-identification/},

year  = {2019},

date = {2019-10-28},

publisher = {International Conference on Computer Vision (ICCV 2019), Seoul, South Korea, Oct 27-Nov 2},

keywords = {Re-Identification},

pubstate = {published},

tppubtype = {conference}

}

@conference{Regmi2019,

title = { Bridging the Domain Gap for Ground-to-Aerial Image Matching},

author = {Krishna Regmi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/02/Publications_Bridging-the-Domain-Gap-for-Ground-to-Aerial-Image-Matching_Final.pdf

https://www.crcv.ucf.edu/research/projects/bridging-the-domain-gap-for-ground-to-aerial-image-matching/

},

year  = {2019},

date = {2019-10-27},

publisher = {International Conference on Computer Vision (ICCV 2019), Seoul, South Korea, Oct 27-Nov 2},

keywords = {Cross-View Image Retrieval, Geo-Localization},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hou2019,

title = { An Efficient 3D CNN for Action/Object Segmentation in Video},

author = {Rui Hou and Chen Chen and Rahul Sukthankar and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2019/07/Publications_An-Efficient-3D-CNN-for-Action-Object-Segmentation-in-Video.pdf

https://www.crcv.ucf.edu/research/projects/an-efficient-3d-cnn-for-action-object-segmentation-in-video/},

year  = {2019},

date = {2019-09-10},

publisher = {British Machine Vision Conference (BMVC 2019), UK, Sep 9-10},

keywords = {BMVC, Video Object Segmentation},

pubstate = {published},

tppubtype = {conference}

}