@conference {5302, title = {The Aligned Multimodal Movie Treebank: An audio, video, dependency-parse treebank}, booktitle = {Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, year = {2022}, abstract = {

Treebanks have traditionally included only text and were derived from written sources such as newspapers or the web. We introduce the Aligned Multimodal Movie Treebank (AMMT), an English language treebank derived from dialog in Hollywood movies which includes transcriptions of the audio-visual streams with word-level alignment, as well as part of speech tags and dependency parses in the Universal Dependencies formalism. AMMT consists of 31,264 sentences and 218,090 words, that will amount to the 3rd largest UD English treebank and the only multimodal treebank in UD. To help with the web-based annotation effort, we also introduce the Efficient Audio Alignment Annotator (EAAA), a companion tool that enables annotators to significantly speed-up their annotation processes.

}, author = {Adam Yaari and Jan DeWitt and Henry Hu and Bennett Stankovits and Sue Felshin and Yevgeni Berzak and Helena Aparicio and Boris Katz and Ignacio Cases and Andrei Barbu} } @conference {4518, title = {Learning Language from Vision.}, booktitle = {Workshop on Visually Grounded Interaction and Language (ViGIL) at the Thirty-third Annual Conference on Neural Information Processing Systems (NeurIPS)}, year = {2019}, month = {12/2019}, address = {Vancouver Convention Center, Vancouver, Canada}, author = {Candace Ross and Yevgeni Berzak and Boris Katz and Andrei Barbu} } @proceedings {3651, title = {Assessing Language Proficiency from Eye Movements in Reading}, year = {2018}, month = {06/2018}, address = {New Orleans}, keywords = {Computation, language}, url = {http://naacl2018.org/}, author = {Yevgeni Berzak and Boris Katz and Roger Levy} } @conference {4109, title = {Grounding language acquisition by training semantic parsersusing captioned videos}, booktitle = {Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP 2018), }, year = {2018}, month = {10/2018 }, address = {Brussels, Belgium}, abstract = {

We develop a semantic parser that is trained ina grounded setting using pairs of videos cap-tioned with sentences. This setting is bothdata-efficient, requiring little annotation, andsimilar to the experience of children wherethey observe their environment and listen tospeakers. The semantic parser recovers themeaning of English sentences despite not hav-ing access to any annotated sentences. It doesso despite the ambiguity inherent in visionwhere a sentence may refer to any combina-tion of objects, object properties, relations oractions taken by any agent in a video. For thistask, we collected a new dataset for groundedlanguage acquisition. Learning a grounded se-mantic parser {\textemdash} turning sentences into logi-cal forms using captioned videos {\textemdash} can sig-nificantly expand the range of data that parserscan be trained on, lower the effort of training asemantic parser, and ultimately lead to a betterunderstanding of child language acquisition.

}, isbn = {978-1-948087-84-1}, url = {http://aclweb.org/anthology/D18-1285}, author = {Candace Ross and Andrei Barbu and Yevgeni Berzak and Battushig Myanganbayar and Boris Katz} } @conference {2587, title = {Predicting Native Language from Gaze}, booktitle = {Annual Meeting of the Association for Computational Linguistics (ACL 2017)}, year = {2017}, author = {Yevgeni Berzak and Chie Nakamura and Suzanne Flynn and Boris Katz} } @article {2214, title = {Anchoring and Agreement in Syntactic Annotations}, year = {2016}, month = {09/2016}, abstract = {

Published in the Proceedings of EMNLP 2016

We present a study on two key characteristics of human syntactic annotations: anchoring and agreement. Anchoring is a well-known cognitive bias in human decision making, where judgments are drawn towards preexisting values. We study the influence of anchoring on a standard approach to creation of syntactic resources where syntactic annotations are obtained via human editing of tagger and parser output. Our experiments demonstrate a clear anchoring effect and reveal unwanted consequences, including overestimation of parsing performance and lower quality of annotations in comparison with human-based annotations. Using sentences from the Penn Treebank WSJ, we also report systematically obtained inter-annotator agreement estimates for English dependency parsing. Our agreement results control for parser bias, and are consequential in that they are on par with state of the art parsing performance for English newswire. We discuss the impact of our findings on strategies for future annotation efforts and parser evaluations.

}, author = {Yevgeni Berzak and Yan Huang and Andrei Barbu and Anna Korhonen and Boris Katz} } @article {2132, title = {Contrastive Analysis with Predictive Power: Typology Driven Estimation of Grammatical Error Distributions in ESL}, year = {2016}, month = {07/2015}, abstract = {

This work examines the impact of crosslinguistic transfer on grammatical errors in English as Second Language (ESL) texts. Using a computational framework that formalizes the theory of Contrastive Analysis (CA), we demonstrate that language specific error distributions in ESL writing can be predicted from the typological properties of the native language and their relation to the typology of English. Our typology driven model enables to obtain accurate estimates of such distributions without access to any ESL data for the target languages. Furthermore, we present a strategy for adjusting our method to low-resource languages that lack typological documentation using a bootstrapping approach which approximates native language typology from ESL texts. Finally, we show that our framework is instrumental for linguistic inquiry seeking to identify first language factors that contribute to a wide range of difficulties in second language acquisition.

}, author = {Yevgeni Berzak and Roi Reichart and Boris Katz} } @article {2133, title = {Do You See What I Mean? Visual Resolution of Linguistic Ambiguities}, year = {2016}, month = {09/2016}, abstract = {

Understanding language goes hand in hand with the ability to integrate complex contextual information obtained via perception. In this work, we present a novel task for grounded language understanding: disambiguating a sentence given a visual scene which depicts one of the possible interpretations of that sentence. To this end, we introduce a new multimodal corpus containing ambiguous sentences, representing a wide range of syntactic, semantic and discourse ambiguities, coupled with videos that visualize the different interpretations for each sentence. We address this task by extending a vision model which determines if a sentence is depicted by a video. We demonstrate how such a model can be adjusted to recognize different interpretations of the same underlying sentence, allowing to disambiguate sentences in a unified fashion across the different ambiguity types.

}, author = {Yevgeni Berzak and Andrei Barbu and Daniel Harari and Boris Katz and Shimon Ullman} } @article {1885, title = {Language and Vision Ambiguities (LAVA) Corpus}, year = {2016}, month = {01/2016}, abstract = {

Ambiguity is one of the defining characteristics of human languages, and language understanding crucially relies on the ability to obtain unambiguous representations of linguistic content. While some ambiguities can be resolved using intra-linguistic contextual cues, the disambiguation of many linguistic constructions requires integration of world knowledge and perceptual information obtained from other modalities. In this work, we focus on the problem of grounding language in the visual modality, and introduce a novel task for visual and linguistic understanding which requires resolving linguistic ambiguities by utilizing the visual context of the utterance.

To address this challenge, we release the Language and Vision Ambiguities (LAVA) corpus. LAVA contains ambiguous sentences coupled with visual scenes that depict the different interpretations of each sentence. The sentences in the corpus are annotated with syntactic and semantic parses, and cover a wide range of linguistic ambiguities, including PP and VP attachment, conjunctions, logical forms, anaphora and ellipsis. In addition to the sentence disambiguation challenge, the corpus will support a variety of related tasks which use natural language as a medium for expressing visual understanding.

Reference:
Yevgeni Berzak, Andrei Barbu, Daniel Harari, Boris Katz, and Shimon Ullman (2015). Do You See What I Mean? Visual Resolution of Linguistic Ambiguities. Conference on Empirical Methods in Natural Language Processing (EMNLP), Lisbon, Portugal. [PDF]

Download all of the clips in MP4 format (ZIP)

}, url = {http://web.mit.edu/lavacorpus/}, author = {Yevgeni Berzak and Andrei Barbu and Daniel Harari and Boris Katz and Shimon Ullman} } @article {2825, title = {Treebank of Learner English (TLE)}, year = {2016}, month = {08/2016}, abstract = {

The majority of the English text available worldwide is generated by non-native speakers. Learner language introduces a variety of challenges\ and is of paramount importance for the scientific study of language acquisition as well as for Natural Language Processing. Despite the ubiquity of non-native English, there has been no publicly available syntactic treebank for English as a Second Language (ESL). To address this shortcoming, we released the Treebank of Learner English (TLE), a first of its kind resource for non-native English, containing 5,124 sentences manually annotated with Part of Speech (POS) tags and syntactic dependency trees. Full syntactic analyses are provided for both the original and error corrected versions of each sentence. We also introduced annotation guidelines that allow for consistent syntactic treatment of ungrammatical English. We envision the treebank to support a wide range of linguistic and computational research on language learning as well as automatic processing of ungrammatical language.

}, url = {http://esltreebank.org/}, author = {Yevgeni Berzak and Jessica Kenney and Carolyn Spadine and Jing Xian Wang and Lucia Lam and Keiko Sophie Mori and Sebastian Garza and Boris Katz} } @article {2134, title = {Universal Dependencies for Learner English}, year = {2016}, month = {06/2016}, abstract = {

We introduce the Treebank of Learner English (TLE), the first publicly available syntactic treebank for English as a Second Language (ESL). The TLE provides manually annotated POS tags and Universal Dependency (UD) trees for 5,124 sentences from the Cambridge First Certificate in English (FCE) corpus. The UD annotations are tied to a pre-existing error annotation of the FCE, whereby full syntactic analyses are provided for both the original and error corrected versions of each sentence. Further on, we delineate ESL annotation guidelines that allow for consistent syntactic treatment of ungrammatical English. Finally, we benchmark POS tagging and dependency parsing performance on the TLE dataset and measure the effect of grammatical errors on parsing accuracy. We envision the treebank to support a wide range of linguistic and computational research o n second language acquisition as well as automatic processing of ungrammatical language.

}, author = {Yevgeni Berzak and Jessica Kenney and Carolyn Spadine and Jing Xian Wang and Lucia Lam and Keiko Sophie Mori and Sebastian Garza and Boris Katz} } @conference {1077, title = {Contrastive Analysis with Predictive Power: Typology Driven Estimation of Grammatical Error Distributions in ESL}, booktitle = {Nineteenth Conference on Computational Natural Language Learning (CoNLL), Beijing, China}, year = {2015}, month = {07/31/2015}, abstract = {

This work examines the impact of cross- linguistic transfer on grammatical errors in English as Second Language (ESL) texts. Using a computational framework that for- malizes the theory of Contrastive Analy- sis (CA), we demonstrate that language specific error distributions in ESL writ- ing can be predicted from the typologi- cal properties of the native language and their relation to the typology of English. Our typology driven model enables to ob- tain accurate estimates of such distribu- tions without access to any ESL data for the target languages. Furthermore, we present a strategy for adjusting our method to low-resource languages that lack typo- logical documentation using a bootstrap- ping approach which approximates native language typology from ESL texts. Fi- nally, we show that our framework is in- strumental for linguistic inquiry seeking to identify first language factors that con- tribute to a wide range of difficulties in second language acquisition

}, author = {Yevgeni Berzak and Roi Reichart and Boris Katz} } @conference {1429, title = {Do You See What I Mean? Visual Resolution of Linguistic Ambiguities}, booktitle = {Conference on Empirical Methods in Natural Language Processing, Lisbon, Portugal. }, year = {2015}, month = {09/2015}, author = {Yevgeni Berzak and Andrei Barbu and Daniel Harari and Boris Katz and Shimon Ullman} } @article {439, title = {Reconstructing Native Language Typology from Foreign Language Usage.}, number = {007}, year = {2014}, month = {04/2014}, abstract = {

Linguists and psychologists have long been studying cross-linguistic transfer, the influence of native language properties on linguistic performance in a foreign language. In this work we provide empirical evidence for this process in the form of a strong correlation between language similarities derived from structural features in English as Second Language (ESL) texts and equivalent similarities obtained directly from the typological features of the native languages. We leverage this finding to recover native language typological similarity structure directly from ESL text, and perform prediction of typological features in an unsupervised fashion with respect to the target languages. Our method achieves 72.2\% accuracy on the typology prediction task, a result that is highly competitive with equivalent methods that rely on typological resources.

}, keywords = {language, linguistics, Visual Intelligence}, author = {Yevgeni Berzak and Roi Reichart and Boris Katz} }