@inproceedings{b0fb7ac675774b7789f81a89176e21e1,
title = "FERNIE-ViL: Facial Expression Enhanced Vision-And-Language Model",
abstract = "Visual cognition requires analyzing actions, intentions, and emotions of persons in a given image. Visual Commonsense Reasoning (VCR) is a task that selects rationales and answers to questions for given images. In VCR, facial expressions are important nonverbal signals because they convey emotions and intentions in human interactions. However, ERNIE-ViL and UNITER, which are vision-And-language models to get image and text representations, do not learn them. We find that ERNIE-ViL and UNITER are vulnerable to the problem of identifying emotions. In this paper, therefore, we propose facial expression recognition FERNIE-ViL, which adapts a facial expression recognition module to the existing vision-And-language model. Experimental results (2.4% point improvement on VCR Q?A and 0.3% point improvement on VCR QA?R) demonstrate that our method can enhance visual commonsense reasoning by understanding human interactions.",
keywords = "Artificial Intelligence, Commonsense Reasoning, Facial Expression, Machine Commonsense, Multi-modal, Natural Language Processing, Visual Recognition",
author = "Lee, {Soo Ryeon} and Dohyun Kim and Mingyu Lee and Sangkeun Lee",
note = "Funding Information: We thank the anonymous reviewers for their helpful comments. This work was supported by the National Research Foundation of Korea (NRF) grant funded by the Korea government (MSIT) (2021R1A2C3010430) and the Basic Research Program through the National Research Foundation of Korea (NRF) grant funded by the Korea government (MSIT) (2020R1A4A1018309). Publisher Copyright: {\textcopyright} 2021 IEEE.; 20th IEEE International Conference on Cognitive Informatics and Cognitive Computing, ICCI*CC 2021 ; Conference date: 29-10-2021 Through 31-10-2021",
year = "2021",
doi = "10.1109/ICCICC53683.2021.9811331",
language = "English",
series = "Proceedings of 2021 IEEE 20th International Conference on Cognitive Informatics and Cognitive Computing, ICCI*CC 2021",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "228--234",
editor = "Yingxu Wang and Wang, {Jane Z.} and Henry Leung and Newton Howard and Paolo Soda and Bernard Widrow and Jerome Feldman",
booktitle = "Proceedings of 2021 IEEE 20th International Conference on Cognitive Informatics and Cognitive Computing, ICCI*CC 2021",
}