generated by jekyll-scholar.
2024
-
Does ChatGPT Have a Poetic Style?Melanie Walsh, Anna Preus, and Elizabeth GronskiIn Computational Humanities Research (CHR), Dec 2024arXiv:2410.15299Generating poetry has become a popular application of LLMs, perhaps especially of OpenAI’s widely-used chatbot ChatGPT. What kind of poet is ChatGPT? Does ChatGPT have its own poetic style? Can it successfully produce poems in different styles? To answer these questions, we prompt the GPT-3.5 and GPT-4 models to generate English-language poems in 24 different poetic forms and styles, about 40 different subjects, and in response to 3 different writing prompt templates. We then analyze the resulting 5.7k poems, comparing them to a sample of 3.7k poems from the Poetry Foundation and the Academy of American Poets. We find that the GPT models, especially GPT-4, can successfully produce poems in a range of both common and uncommon English-language forms in superficial yet noteworthy ways, such as by producing poems of appropriate lengths for sonnets (14 lines), villanelles (19 lines), and sestinas (39 lines). But the GPT models also exhibit their own distinct stylistic tendencies, both within and outside of these specific forms. Our results show that GPT poetry is much more constrained and uniform than human poetry, showing a strong penchant for rhyme, quatrains (4-line stanzas), iambic meter, first-person plural perspectives (we, us, our), and specific vocabulary like “heart,” “embrace,” “echo,” and “whisper.”
@inproceedings{Walsh_Preus_Gronski_2024, title = {Does ChatGPT Have a Poetic Style?}, doi = {10.48550/arXiv.2410.15299}, note = {arXiv:2410.15299}, number = {arXiv:2410.15299}, publisher = {arXiv}, author = {Walsh, Melanie and Preus, Anna and Gronski, Elizabeth}, year = {2024}, month = dec, booktitle = {Computational Humanities Research (CHR)} } -
Sonnet or Not, Bot? Poetry Evaluation for Large Models and DatasetsMelanie Walsh, Maria Antoniak, and Anna PreusIn Findings of the Association for Computational Linguistics: EMNLP, Nov 2024Large language models (LLMs) can now generate and recognize poetry. But what do LLMs really know about poetry? We develop a task to evaluate how well LLMs recognize one aspect of English-language poetry—poetic form—which captures many different poetic features, including rhyme scheme, meter, and word or line repetition. By using a benchmark dataset of over 4.1k human expert-annotated poems, we show that state-of-the-art LLMs can successfully identify both common and uncommon fixed poetic forms—such as sonnets, sestinas, and pantoums—with surprisingly high accuracy. However, performance varies significantly by poetic form; the models struggle to identify unfixed poetic forms, especially those based on topic or visual features. We additionally measure how many poems from our benchmark dataset are present in popular pretraining datasets or memorized by GPT-4, finding that pretraining presence and memorization may improve performance on this task, but results are inconclusive. We release a benchmark evaluation dataset with 1.4k public domain poems and form annotations, results of memorization experiments and data audits, and code.
@inproceedings{Walsh_Antoniak_Preus_2024, title = {Sonnet or Not, Bot? Poetry Evaluation for Large Models and Datasets}, abstractnote = {Large language models (LLMs) can now generate and recognize poetry. But what do LLMs really know about poetry? We develop a task to evaluate how well LLMs recognize one aspect of English-language poetry—poetic form—which captures many different poetic features, including rhyme scheme, meter, and word or line repetition. By using a benchmark dataset of over 4.1k human expert-annotated poems, we show that state-of-the-art LLMs can successfully identify both common and uncommon fixed poetic forms—such as sonnets, sestinas, and pantoums—with surprisingly high accuracy. However, performance varies significantly by poetic form; the models struggle to identify unfixed poetic forms, especially those based on topic or visual features. We additionally measure how many poems from our benchmark dataset are present in popular pretraining datasets or memorized by GPT-4, finding that pretraining presence and memorization may improve performance on this task, but results are inconclusive. We release a benchmark evaluation dataset with 1.4k public domain poems and form annotations, results of memorization experiments and data audits, and code.}, booktitle = {Findings of the Association for Computational Linguistics: EMNLP}, author = {Walsh, Melanie and Antoniak, Maria and Preus, Anna}, year = {2024}, month = nov, language = {en} } -
The Afterlives of Shakespeare and Company in Online Social ReadershipMaria Antoniak, David Mimno, Rosamond Thalken, Melanie Walsh, Matthew Wilkens, and Gregory YauneyModernism/modernity and Journal of Cultural Analytics, Jan 2024Modernism/modernity Journal of Cultural Analytics arXiv PDF Interactive Plots Code & Data Copy Bibtex Citation Read the AbstractThe growth of social reading platforms such as Goodreads and LibraryThing enables us to analyze reading activity at very large scale and in remarkable detail. But twenty-first century systems give us a perspective only on contemporary readers. Meanwhile, the digitization of the lending library records of Shakespeare and Company provides a window into the reading activity of an earlier, smaller community in interwar Paris. In this article, we explore the extent to which we can make comparisons between the Shakespeare and Company and Goodreads communities. By quantifying similarities and differences, we can identify patterns in how works have risen or fallen in popularity across these datasets. We can also measure differences in how works are received by measuring similarities and differences in co-reading patterns. Finally, by examining the complete networks of co-readership, we can observe changes in the overall structures of literary reception.
@article{Antoniak_Mimno_Thalken_Walsh_Wilkens_Yauney_2024, title = {The Afterlives of Shakespeare and Company in Online Social Readership}, url = {http://arxiv.org/abs/2401.07340}, modernism = {https://modernismmodernity.org/forums/posts/antoniak-etal-afterlives-shakespeare-co-online-social-readership}, abstractnote = {The growth of social reading platforms such as Goodreads and LibraryThing enables us to analyze reading activity at very large scale and in remarkable detail. But twenty-first century systems give us a perspective only on contemporary readers. Meanwhile, the digitization of the lending library records of Shakespeare and Company provides a window into the reading activity of an earlier, smaller community in interwar Paris. In this article, we explore the extent to which we can make comparisons between the Shakespeare and Company and Goodreads communities. By quantifying similarities and differences, we can identify patterns in how works have risen or fallen in popularity across these datasets. We can also measure differences in how works are received by measuring similarities and differences in co-reading patterns. Finally, by examining the complete networks of co-readership, we can observe changes in the overall structures of literary reception.}, number = {arXiv:2401.07340}, journal = {Modernism/modernity and Journal of Cultural Analytics}, publisher = {Johns Hopkins University Press}, author = {Antoniak, Maria and Mimno, David and Thalken, Rosamond and Walsh, Melanie and Wilkens, Matthew and Yauney, Gregory}, year = {2024}, month = jan }
2023
-
Riveter: Measuring Power and Social Dynamics Between EntitiesMaria Antoniak, Anjalie Field, Jimin Mun, Melanie Walsh, Lauren F. Klein, and Maarten SapIn Association for Computational Linguistics (ACL), Jan 2023arXiv:2312.09536 [cs]Riveter provides a complete easy-to-use pipeline for analyzing verb connotations associated with entities in text corpora. We prepopulate the package with connotation frames of sentiment, power, and agency, which have demonstrated usefulness for capturing social phenomena, such as gender bias, in a broad range of corpora. For decades, lexical frameworks have been foundational tools in computational social science, digital humanities, and natural language processing, facilitating multifaceted analysis of text corpora. But working with verb-centric lexica specifically requires natural language processing skills, reducing their accessibility to other researchers. By organizing the language processing pipeline, providing complete lexicon scores and visualizations for all entities in a corpus, and providing functionality for users to target specific research questions, Riveter greatly improves the accessibility of verb lexica and can facilitate a broad range of future research.
@inproceedings{Antoniak_Field_Mun_Walsh_Klein_Sap_2023, title = {Riveter: Measuring Power and Social Dynamics Between Entities}, url = {http://arxiv.org/abs/2312.09536}, doi = {10.18653/v1/2023.acl-demo.36}, acl = {https://aclanthology.org/2023.acl-demo.36/}, abstractnote = {Riveter provides a complete easy-to-use pipeline for analyzing verb connotations associated with entities in text corpora. We prepopulate the package with connotation frames of sentiment, power, and agency, which have demonstrated usefulness for capturing social phenomena, such as gender bias, in a broad range of corpora. For decades, lexical frameworks have been foundational tools in computational social science, digital humanities, and natural language processing, facilitating multifaceted analysis of text corpora. But working with verb-centric lexica specifically requires natural language processing skills, reducing their accessibility to other researchers. By organizing the language processing pipeline, providing complete lexicon scores and visualizations for all entities in a corpus, and providing functionality for users to target specific research questions, Riveter greatly improves the accessibility of verb lexica and can facilitate a broad range of future research.}, note = {arXiv:2312.09536 [cs]}, booktitle = {Association for Computational Linguistics (ACL)}, author = {Antoniak, Maria and Field, Anjalie and Mun, Jimin and Walsh, Melanie and Klein, Lauren F. and Sap, Maarten}, year = {2023}, pages = {377–388} } -
The Challenges and Possibilities of Social Media Data: New Directions in Literary Studies and the Digital HumanitiesMelanie WalshJan 2023publisher: U of Minnesota Press@inbook{Walsh_2023, title = {The Challenges and Possibilities of Social Media Data: New Directions in Literary Studies and the Digital Humanities}, url = {https://dhdebates.gc.cuny.edu/read/f3f87448-138c-4d19-8ff8-b06acf40ddd1/section/a57b98ab-0f10-45d0-b205-3e563aab7ea8#ch18}, note = {publisher: U of Minnesota Press}, booktitle = {Debates in the Digital Humanities 2023}, publisher = {University of Minnesota Press}, author = {Walsh, Melanie}, year = {2023} }
2022
-
Where is all the book data?Melanie WalshPublic Books, Jan 2022🎙️I spoke about this essay with Australian radio ABC.🎙️
2021
-
Tags, Borders, and Catalogs: Social Re-Working of Genre on LibraryThingMaria Antoniak, Melanie Walsh, and David MimnoCSCW, Apr 2021Through a computational reading of the online book reviewing community LibraryThing, we examine the dynamics of a collaborative tagging system and learn how its users refine and redefine literary genres. LibraryThing tags are overlapping and multi-dimensional, created in a shared space by thousands of users, including readers, bookstore owners, and librarians. A common understanding of genre is that it relates to the content of books, but this resource allows us to view genre as an intersection of user communities and reader values and interests. We explore different methods of computational genre measurement within the open space of user-created tags. We measure overlap between books, tags, and users, and we also measure the homogeneity of communities associated with genre tags and correlate this homogeneity with reviewing behavior.Finally, by analyzing the text of reviews, we identify the thematic signatures of genres on LibraryThing, revealing similarities and differences between them. These measurements are intended to elucidate the genre conceptions of the users, not, as in prior work, to normalize the tags or enforce a hierarchy. We find that LibraryThing users make sense of genre through a variety of values and expectations, many of which fall outside common definitions and understandings of genre.
@article{Antoniak_Walsh_Mimno_2021, title = {Tags, Borders, and Catalogs: Social Re-Working of Genre on LibraryThing}, volume = {5}, issn = {2573-0142}, doi = {10.1145/3449103}, abstractnote = {Through a computational reading of the online book reviewing community LibraryThing, we examine the dynamics of a collaborative tagging system and learn how its users refine and redefine literary genres. LibraryThing tags are overlapping and multi-dimensional, created in a shared space by thousands of users, including readers, bookstore owners, and librarians. A common understanding of genre is that it relates to the content of books, but this resource allows us to view genre as an intersection of user communities and reader values and interests. We explore different methods of computational genre measurement within the open space of user-created tags. We measure overlap between books, tags, and users, and we also measure the homogeneity of communities associated with genre tags and correlate this homogeneity with reviewing behavior.Finally, by analyzing the text of reviews, we identify the thematic signatures of genres on LibraryThing, revealing similarities and differences between them. These measurements are intended to elucidate the genre conceptions of the users, not, as in prior work, to normalize the tags or enforce a hierarchy. We find that LibraryThing users make sense of genre through a variety of values and expectations, many of which fall outside common definitions and understandings of genre.}, url = {https://dl.acm.org/doi/10.1145/3449103}, number = {CSCW}, journal = {CSCW}, author = {Antoniak, Maria and Walsh, Melanie and Mimno, David}, year = {2021}, month = apr, pages = {1–29}, language = {en} } -
The Goodreads “Classics”: A Computational Study of Readers, Amazon, and Crowdsourced Amateur CriticismMelanie Walsh, and Maria AntoniakPost45 and Journal of Cultural Analytics, Apr 2021Post45 Journal of Cultural Analytics PDF Interactive Plots Code & Data Goodreads Scraper Copy Bibtex Citation@article{Walsh_Antoniak_2021, title = {The Goodreads “Classics”: A Computational Study of Readers, Amazon, and Crowdsourced Amateur Criticism}, volume = {6}, url = {https://culturalanalytics.org/article/22221.pdf}, number = {2}, journal = {Post45 and Journal of Cultural Analytics}, author = {Walsh, Melanie and Antoniak, Maria}, year = {2021}, bibtex_show = true } -
Introduction to Cultural Analytics & PythonMelanie WalshApr 2021🏆 Voted Best Digital Humanities Training Material in 2021 🏆Voted Best Digital Humanities Training Material in 2021
@book{Walsh_2021, title = {Introduction to Cultural Analytics & Python}, url = {https://melaniewalsh.github.io/Intro-Cultural-Analytics}, publisher = {Jupyter Book}, author = {Walsh, Melanie}, year = {2021} }
2018
-
Tweets of a Native Son: The Quotation and Recirculation of James Baldwin from Black Power to #BlackLivesMatterMelanie WalshAmerican Quarterly, Apr 2018@article{Walsh_2018, title = {Tweets of a Native Son: The Quotation and Recirculation of James Baldwin from Black Power to #BlackLivesMatter}, volume = {70}, number = {3}, journal = {American Quarterly}, publisher = {Johns Hopkins University Press}, author = {Walsh, Melanie}, year = {2018}, pages = {531–559}, }