included_papers.bib

% BibTeX entries for 71 papers in systematic literature review

@inproceedings{adamkiewicz2025,
author = {Adamkiewicz, Krzysztof and Wo\'{z}niak, Pawe\l{} W. and Dominiak, Julia and Romanowski, Andrzej and Karolus, Jakob and Frolov, Stanislav},
title = {PromptMap: An Alternative Interaction Style for AI-Based Image Generation},
year = {2025},
isbn = {9798400713064},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3708359.3712150},
doi = {10.1145/3708359.3712150},
abstract = {Recent technological advances popularized the use of image generation among the general public. Crafting effective prompts can, however, be difficult for novice users. To tackle this challenge, we developed PromptMap, a new interaction style for text-to-image AI that allows users to freely explore a vast collection of synthetic prompts through a map-like view with semantic zoom. PromptMap groups images visually by their semantic similarity, allowing users to discover relevant examples. We evaluated PromptMap in a between-subject online study (n = 60) and a qualitative within-subject study (n = 12). We found that PromptMap supported users in crafting prompts by providing them with examples. We also demonstrated the feasibility of using LLMs to create vast example collections. Our work contributes a new interaction style that supports users unfamiliar with prompting in achieving a satisfactory image output.},
booktitle = {Proceedings of the 30th International Conference on Intelligent User Interfaces},
pages = {1162–1176},
numpages = {15},
keywords = {Generative AI, image generation, interaction methods},
location = {
},
series = {IUI '25}
}

@inproceedings{almeda2024,
author = {Almeda, Shm Garanganao and Zamfirescu-Pereira, J.D. and Kim, Kyu Won and Mani Rathnam, Pradeep and Hartmann, Bjoern},
title = {Prompting for Discovery: Flexible Sense-Making for AI Art-Making with Dreamsheets},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642858},
doi = {10.1145/3613904.3642858},
abstract = {Design space exploration (DSE) for Text-to-Image (TTI) models entails navigating a vast, opaque space of possible image outputs, through a commensurately vast input space of hyperparameters and prompt text. Perceptually small movements in prompt-space can surface unexpectedly disparate images. How can interfaces support end-users in reliably steering prompt-space explorations towards interesting results? Our design probe, DreamSheets, supports user-composed exploration strategies with LLM-assisted prompt construction and large-scale simultaneous display of generated results, hosted in a spreadsheet interface. Two studies, a preliminary lab study and an extended two-week study where five expert artists developed custom TTI sheet-systems, reveal various strategies for targeted TTI design space exploration—such as using templated text generation to define and layer semantic “axes” for exploration. We identified patterns in exploratory structures across our participants’ sheet-systems: configurable exploration “units” that we distill into a UI mockup, and generalizable UI components to guide future interfaces.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {160},
numpages = {17},
keywords = {design space exploration, generative AI, text to image},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@article{arakawa2024,
author = {Arakawa, Riku and Maeda, Kiyosu and Yakura, Hiromu},
title = {ConverSearch: Supporting Experts in Human Behavior Analysis of Conversational Videos with a Multimodal Scene Search Tool},
year = {2025},
issue_date = {March 2025},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {15},
number = {1},
issn = {2160-6455},
url = {https://doi.org/10.1145/3709012},
doi = {10.1145/3709012},
abstract = {Multimodal scene search of conversations is essential for unlocking valuable insights into social dynamics and enhancing our communication. While experts in conversational analysis have their own knowledge and skills to find key scenes, a lack of comprehensive, user-friendly tools that streamline the processing of diverse multimodal queries impedes efficiency and objectivity. To address this gap, we developed ConverSearch, a visual-programming-based tool based on insights for effective interface and implementation design derived from a formative study with experts. The tool allows experts to integrate various machine learning algorithms to capture human behavioral cues without the need for coding. Our user study, employing the System Usability Scale (SUS) and satisfaction metrics, demonstrated high user preference, reflecting the tool’s ease of use and effectiveness in supporting scene search tasks. Additionally, through a deployment trial within industrial organizations, we confirmed the tool’s objectivity, reusability, and potential to enhance expert workflows. This suggests the advantages of expert-AI collaboration in domains requiring human contextual understanding and demonstrates how customizable, transparent tools yielding reusable artifacts can support expert-driven tasks in complex, multimodal environments.},
journal = {ACM Trans. Interact. Intell. Syst.},
month = feb,
articleno = {6},
numpages = {31},
keywords = {behavior analysis, scene search, visual programming, expert-AI collaboration}
}

@inproceedings{barnaby2024,
author = {Barnaby, Celeste and Chen, Qiaochu and Wang, Chenglong and Dillig, Isil},
title = {PhotoScout: Synthesis-Powered Multi-Modal Image Search},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642319},
doi = {10.1145/3613904.3642319},
abstract = {Due to the availability of increasingly large amounts of visual data, there is a growing need for tools that can help users find relevant images. While existing tools can perform image retrieval based on similarity or metadata, they fall short in scenarios that necessitate semantic reasoning about the content of the image. This paper explores a new multi-modal image search approach that allows users to conveniently specify and perform semantic image search tasks. With our tool, PhotoScout, the user interactively provides natural language descriptions, positive and negative examples, and object tags to specify their search tasks. Under the hood, PhotoScout is powered by a program synthesis engine that generates visual queries in a domain-specific language and executes the synthesized program to retrieve the desired images. In a study with 25 participants, we observed that PhotoScout allows users to perform image retrieval tasks more accurately and with less manual effort.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {896},
numpages = {15},
keywords = {Interface design, multi-modal interfaces, program synthesis},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{bourgault2025,
author = {Bourgault, Samuelle and Wei, Li-Yi and Jacobs, Jennifer and Kazi, Rubaiat Habib},
title = {Narrative Motion Blocks: Combining Direct Manipulation and Natural Language Interactions for Animation Creation},
year = {2025},
isbn = {9798400714856},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3715336.3735766},
doi = {10.1145/3715336.3735766},
abstract = {Authoring compelling animations often requires artists to come up with creative high-level ideas and translate them into precise low-level spatial and temporal properties like position, orientation, scale, and frame timing. Traditional animation tools offer direct manipulation strategies to control these properties but lack support for implementing higher-level ideas. Alternatively, AI-based tools allow animation production using natural language prompts but lack the fine-grained control over properties required for professional workflows. To bridge this gap, we propose AniMate, a hand-drawn animation system that integrates direct manipulation and natural language interaction. Central to AniMate are narrative motion blocks, clip-like components located on a timeline that let animators specify animated behaviors with a combination of textual and manual input. Through an expert evaluation and the creation of short demonstrative animations, we show how focusing on intermediate-level actions provides a common representation for animators to work across both interaction modalities.},
booktitle = {Proceedings of the 2025 ACM Designing Interactive Systems Conference},
pages = {1366–1386},
numpages = {21},
keywords = {Creativity-Support Tool, Animation, Direct Manipulation, Natural Language Interaction, LLM},
location = {
},
series = {DIS '25}
}

@inproceedings{boyd2023,
author = {Boyd, Kyle and McAllister, Patrick and Mulvenna, Maurice and Bond, Raymond and Wang, Hui and Spence, Ivor and Wu, Guanfeng and Haider, Abbas},
title = {Designing Multimodal Video Search by Examples (MVSE) user interfaces: UX requirements elicitation and insights from semi-structured interviews},
year = {2023},
isbn = {9798400708756},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3605655.3605665},
doi = {10.1145/3605655.3605665},
abstract = {In order to search for content from large video archives, it is typically undertaken via keyword queries using predefined metadata such as title and other tags. However, it is difficult to use keywords to search for specific moments in a video. Video search by examples is a desirable approach for this scenario as it allows users to search for content using one or more examples without having to specify a keyword. However, video search by examples is notoriously challenging, and performance is poor. To improve search performance, multiple modalities may be considered – image, sound, voice and text, multiple search cues could be used to identify more relevant content. This is multimodal video search by examples (MVSE), where users can search for content using multiple modalities. In this paper, typical end users - BBC archivists, programme support staff - are interviewed to identify how their search needs can be addressed with the technical capabilities of a MVSE tool. Such a search tool will be useful for organisations such as the BBC who maintain large collections of video archives and want to provide a search tool for their own staff as well as for the public. It will also be useful for companies such as Youtube who host videos from the public and want to enable video search by examples. The study’s objectives explored in this paper were to inform the design and development of the UX workflows to gain a broader understanding of what opportunities and issues may arise from the proposed prototype tool. Results from the thematic analysis was highlighted 4 main themes: Opportunities, Time constraints, Activities, and Pain points. Further analysis highlighted key areas that should be considered for an MVSE-based system, such as scene recognition, face recognition, speed issues, and integration..},
booktitle = {Proceedings of the European Conference on Cognitive Ergonomics 2023},
articleno = {4},
numpages = {8},
keywords = {UX, Semi-structured interviews, Requirements elicitation, Multimodal video search, Multimodal Video Search by Examples, Interfaces},
location = {Swansea, United Kingdom},
series = {ECCE '23}
}

@inproceedings{brade2023,
author = {Brade, Stephen and Wang, Bryan and Sousa, Mauricio and Oore, Sageev and Grossman, Tovi},
title = {Promptify: Text-to-Image Generation through Interactive Prompt Exploration with Large Language Models},
year = {2023},
isbn = {9798400701320},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3586183.3606725},
doi = {10.1145/3586183.3606725},
abstract = {Text-to-image generative models have demonstrated remarkable capabilities in generating high-quality images based on textual prompts. However, crafting prompts that accurately capture the user’s creative intent remains challenging. It often involves laborious trial-and-error procedures to ensure that the model interprets the prompts in alignment with the user’s intention. To address these challenges, we present Promptify, an interactive system that supports prompt exploration and refinement for text-to-image generative models. Promptify utilizes a suggestion engine powered by large language models to help users quickly explore and craft diverse prompts. Our interface allows users to organize the generated images flexibly, and based on their preferences, Promptify suggests potential changes to the original prompt. This feedback loop enables users to iteratively refine their prompts and enhance desired features while avoiding unwanted ones. Our user study shows that Promptify effectively facilitates the text-to-image workflow, allowing users to create visually appealing images on their first attempt while requiring significantly less cognitive load than a widely-used baseline tool.},
booktitle = {Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology},
articleno = {96},
numpages = {14},
keywords = {Large Language Models, Prompt Engineering, Text-to-Image},
location = {San Francisco, CA, USA},
series = {UIST '23}
}

@inproceedings{brade2024,
author = {Brade, Stephen and Wang, Bryan and Sousa, Mauricio and Newsome, Gregory Lee and Oore, Sageev and Grossman, Tovi},
title = {SynthScribe: Deep Multimodal Tools for Synthesizer Sound Retrieval and Exploration},
year = {2024},
isbn = {9798400705083},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3640543.3645158},
doi = {10.1145/3640543.3645158},
abstract = {Synthesizers are powerful tools that allow musicians to create dynamic and original sounds. Existing commercial interfaces for synthesizers typically require musicians to interact with complex low-level parameters or to manage large libraries of premade sounds. To address these challenges, we implement SynthScribe — a fullstack system that uses multimodal deep learning to let users express their intentions at a much higher level. We implement features which address a number of difficulties, namely 1) searching through existing sounds, 2) creating completely new sounds, and 3) making meaningful modifications to a given sound. This is achieved with three main features: a multimodal search engine for a large library of synthesizer sounds; a user centered genetic algorithm by which completely new sounds can be created and selected given the users preferences; a sound editing support feature which highlights and gives examples for key control parameters with respect to a text or audio based query. The results of our user studies show SynthScribe is capable of reliably retrieving and modifying sounds while also affording the ability to create completely new sounds that expand a musicians creative horizon.},
booktitle = {Proceedings of the 29th International Conference on Intelligent User Interfaces},
pages = {51–65},
numpages = {15},
keywords = {Multimodal Deep Learning, Sound Editing, Sound Synthesis},
location = {Greenville, SC, USA},
series = {IUI '24}
}

@inproceedings{chakrabarty2024,
author = {Chakrabarty, Goirik and Chandrasekar, Aditya and Hebbalaguppe, Ramya and AP, Prathosh},
title = {LoMOE: Localized Multi-Object Editing via Multi-Diffusion},
year = {2024},
isbn = {9798400706868},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3664647.3681199},
doi = {10.1145/3664647.3681199},
abstract = {Recent developments in diffusion models have demonstrated an exceptional capacity to generate high-quality, prompt-conditioned image edits. Nevertheless, previous approaches have primarily relied on textual prompts for image editing, which tend to be less effective when making precise edits to specific objects or fine-grained regions within a scene containing single/multiple objects. We introduce a novel framework for zero-shot localized multi-object editing through a multi-diffusion process to overcome this challenge. This framework empowers users to perform various operations on objects within an image, such as adding, replacing, or editing many objects in a complex scene in one pass. Our approach leverages foreground masks and corresponding simple text prompts that exert localized influences on the target regions resulting in high-fidelity image editing. A combination of cross-attention and background preservation losses within the latent space ensures that the characteristics of the object being edited are preserved while simultaneously achieving a high-quality, seamless reconstruction of the background with fewer artifacts compared to the state-of-the-art (SOTA). We also curate and release a dataset dedicated to multi-object editing, named LoMOE-Bench. Our experiments against existing SOTA demonstrate the improved effectiveness of our approach in terms of both image editing quality, and inference speed.},
booktitle = {Proceedings of the 32nd ACM International Conference on Multimedia},
pages = {3342–3351},
numpages = {10},
keywords = {diffusion models, image editing, localized editing, multi-object editing, text2image models, vision language models},
location = {Melbourne VIC, Australia},
series = {MM '24}
}

@inproceedings{chang2024,
author = {Chang, Ruei-Che and Liu, Yuxuan and Zhang, Lotus and Guo, Anhong},
title = {EditScribe: Non-Visual Image Editing with Natural Language Verification Loops},
year = {2024},
isbn = {9798400706776},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3663548.3675599},
doi = {10.1145/3663548.3675599},
abstract = {Image editing is an iterative process that requires precise visual evaluation and manipulation for the output to match the editing intent. However, current image editing tools do not provide accessible interaction nor sufficient feedback for blind and low vision individuals to achieve this level of control. To address this, we developed EditScribe, a prototype system that makes object-level image editing actions accessible using natural language verification loops powered by large multimodal models. Using EditScribe, the user first comprehends the image content through initial general and object descriptions, then specifies edit actions using open-ended natural language prompts. EditScribe performs the image edit, and provides four types of verification feedback for the user to verify the performed edit, including a summary of visual changes, AI judgement, and updated general and object descriptions. The user can ask follow-up questions to clarify and probe into the edits or verification feedback, before performing another edit. In a study with ten blind or low-vision users, we found that EditScribe supported participants to perform and verify image edit actions non-visually. We observed different prompting strategies from participants, and their perceptions on the various types of verification feedback. Finally, we discuss the implications of leveraging natural language verification loops to make visual authoring non-visually accessible.},
booktitle = {Proceedings of the 26th International ACM SIGACCESS Conference on Computers and Accessibility},
articleno = {65},
numpages = {19},
keywords = {Accessibility, assistive technology, blind, creativity support tools, generative AI, image editing, low vision, visual authoring},
location = {St. John's, NL, Canada},
series = {ASSETS '24}
}

@inproceedings{chen2023a,
author = {Chen, Yang and Pan, Yingwei and Li, Yehao and Yao, Ting and Mei, Tao},
title = {Control3D: Towards Controllable Text-to-3D Generation},
year = {2023},
isbn = {9798400701085},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3581783.3612489},
doi = {10.1145/3581783.3612489},
abstract = {Recent remarkable advances in large-scale text-to-image diffusion models have inspired a significant breakthrough in text-to-3D generation, pursuing 3D content creation solely from a given text prompt. However, existing text-to-3D techniques lack a crucial ability in the creative process: interactively control and shape the synthetic 3D contents according to users' desired specifications (e.g., sketch). To alleviate this issue, we present the first attempt for text-to-3D generation conditioning on the additional hand-drawn sketch, namely Control3D, which enhances controllability for users. In particular, a 2D conditioned diffusion model (ControlNet) is remoulded to guide the learning of 3D scene parameterized as NeRF, encouraging each view of 3D scene aligned with the given text prompt and hand-drawn sketch. Moreover, we exploit a pre-trained differentiable photo-to-sketch model to directly estimate the sketch of the rendered image over synthetic 3D scene. Such estimated sketch along with each sampled view is further enforced to be geometrically consistent with the given sketch, pursuing better controllable text-to-3D generation. Through extensive experiments, we demonstrate that our proposal can generate accurate and faithful 3D scenes that align closely with the input text prompts and sketches.},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {1148–1156},
numpages = {9},
keywords = {diffusion model, sketch, text-to-3d generation},
location = {Ottawa ON, Canada},
series = {MM '23}
}

@inproceedings{chen2023b,
author = {Chen, Jingwen and Pan, Yingwei and Yao, Ting and Mei, Tao},
title = {ControlStyle: Text-Driven Stylized Image Generation Using Diffusion Priors},
year = {2023},
isbn = {9798400701085},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3581783.3612524},
doi = {10.1145/3581783.3612524},
abstract = {Recently, the multimedia community has witnessed the rise of diffusion models trained on large-scale multi-modal data for visual content creation, particularly in the field of text-to-image generation. In this paper, we propose a new task for "stylizing'' text-to-image models, namely text-driven stylized image generation, that further enhances editability in content creation. Given input text prompt and style image, this task aims to produce stylized images which are both semantically relevant to input text prompt and meanwhile aligned with the style image in style. To achieve this, we present a new diffusion model (ControlStyle) via upgrading a pre-trained text-to-image model with a trainable modulation network enabling more conditions of text prompts and style images. Moreover, diffusion style and content regularizations are simultaneously introduced to facilitate the learning of this modulation network with these diffusion priors, pursuing high-quality stylized text-to-image generation. Extensive experiments demonstrate the effectiveness of our ControlStyle in producing more visually pleasing and artistic results, surpassing a simple combination of text-to-image model and conventional style transfer techniques.},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {7540–7548},
numpages = {9},
keywords = {diffusion models, style transfer, text-to-image generation},
location = {Ottawa ON, Canada},
series = {MM '23}
}

@inproceedings{chen2024b,
author = {Chen, Liuqing and Jing, Qianzhi and Tsang, Yixin and Wang, Qianyi and Liu, Ruocong and Xia, Duowei and Zhou, Yunzhan and Sun, Lingyun},
title = {AutoSpark: Supporting Automobile Appearance Design Ideation with Kansei Engineering and Generative AI},
year = {2024},
isbn = {9798400706288},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3654777.3676337},
doi = {10.1145/3654777.3676337},
abstract = {Rapid creation of novel product appearance designs that align with consumer emotional requirements poses a significant challenge. Text-to-image models, with their excellent image generation capabilities, have demonstrated potential in providing inspiration to designers. However, designers still encounter issues including aligning emotional needs, expressing design intentions, and comprehending generated outcomes in practical applications. To address these challenges, we introduce AutoSpark, an interactive system that integrates Kansei Engineering and generative AI to provide creativity support for designers in creating automobile appearance designs that meet emotional needs. AutoSpark employs a Kansei Engineering engine powered by generative AI and a semantic network to assist designers in emotional need alignment, design intention expression, and prompt crafting. It also facilitates designers’ understanding and iteration of generated results through fine-grained image-image similarity comparisons and text-image relevance assessments. The design-thinking map within its interface aids in managing the design process. Our user study indicates that AutoSpark effectively aids designers in producing designs that are more aligned with emotional needs and of higher quality compared to a baseline system, while also enhancing the designers’ experience in the human-AI co-creation process.},
booktitle = {Proceedings of the 37th Annual ACM Symposium on User Interface Software and Technology},
articleno = {108},
numpages = {19},
keywords = {Creativity Support Tool, Generative AI, Product Appearance Design Ideation},
location = {Pittsburgh, PA, USA},
series = {UIST '24}
}

@inproceedings{chen2024c,
author = {Chen, Cheng and Lee, Sangwook and Jang, Eunchae and Sundar, S. Shyam},
title = {Is Your Prompt Detailed Enough? Exploring the Effects of Prompt Coaching on Users' Perceptions, Engagement, and Trust in Text-to-Image Generative AI Tools},
year = {2024},
isbn = {9798400709890},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3686038.3686060},
doi = {10.1145/3686038.3686060},
abstract = {Prompts are the primary medium for interacting with generative AI tools. However, users often lack sufficient prompt literacy and motivation to fully benefit from these tools. To address this, we explore whether introducing prompt coaching into a chatbot-based generative AI interface can influence users’ perceptions and engagement of prompting, and further affect their trust in the system. In a user study (N = 132), we found that prompt coaching encourages users to specify more details in their prompts, even though over half initially believed their prompts were sufficient. Furthermore, the coach increased users’ cognitive elaboration, which was associated with higher perceived trust calibration. However, prompt coaching did not significantly enhance UX, although users in the coaching absent condition expressed a strong need for prompt assistance for better user experience. These findings have practical implications for the design of trustworthy and responsible generative AI interfaces.},
booktitle = {Proceedings of the Second International Symposium on Trustworthy Autonomous Systems},
articleno = {9},
numpages = {12},
keywords = {Cognitive elaboration, Generative AI, Perceived trust calibration, Prompt coaching, User Engagement, User Experience, User Interface},
location = {Austin, TX, USA},
series = {TAS '24}
}

@inproceedings{christen2024,
author = {Christen, Sammy and Hampali, Shreyas and Sener, Fadime and Remelli, Edoardo and Hodan, Tomas and Sauser, Eric and Ma, Shugao and Tekin, Bugra},
title = {DiffH2O: Diffusion-Based Synthesis of Hand-Object Interactions from Textual Descriptions},
year = {2024},
isbn = {9798400711312},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3680528.3687563},
doi = {10.1145/3680528.3687563},
abstract = {We introduce DiffH2O, a new diffusion-based framework for synthesizing realistic, dexterous hand-object interactions from natural language. Our model employs a temporal two-stage diffusion process, dividing hand-object motion generation into grasping and interaction stages to enhance generalization to various object shapes and textual prompts. To improve generalization to unseen objects and increase output controllability, we propose grasp guidance, which directs the diffusion model towards a target grasp, seamlessly connecting the grasping and interaction stages through a motion imputation mechanism. We demonstrate the practical value of grasp guidance using hand poses extracted from images or grasp synthesis methods. Additionally, we provide detailed textual descriptions for the GRAB dataset, enabling fine-grained text-based control of the model output. Our quantitative and qualitative evaluations show that DiffH2O generates realistic hand-object motions from natural language, generalizes to unseen objects, and significantly outperforms existing methods on a standard benchmark and in perceptual studies.},
booktitle = {SIGGRAPH Asia 2024 Conference Papers},
articleno = {145},
numpages = {11},
keywords = {motion generation, dexterous manipulation, hand-object interaction, diffusion models},
location = {Tokyo, Japan},
series = {SA '24}
}

@inproceedings{chung2023,
author = {Chung, John Joon Young and Adar, Eytan},
title = {PromptPaint: Steering Text-to-Image Generation Through Paint Medium-like Interactions},
year = {2023},
isbn = {9798400701320},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3586183.3606777},
doi = {10.1145/3586183.3606777},
abstract = {While diffusion-based text-to-image (T2I) models provide a simple and powerful way to generate images, guiding this generation remains a challenge. For concepts that are difficult to describe through language, users may struggle to create prompts. Moreover, many of these models are built as end-to-end systems, lacking support for iterative shaping of the image. In response, we introduce PromptPaint, which combines T2I generation with interactions that model how we use colored paints. PromptPaint allows users to go beyond language to mix prompts that express challenging concepts. Just as we iteratively tune colors through layered placements of paint on a physical canvas, PromptPaint similarly allows users to apply different prompts to different canvas areas and times of the generative process. Through a set of studies, we characterize different approaches for mixing prompts, design trade-offs, and socio-technical challenges for generative models. With PromptPaint we provide insight into future steerable generative tools.},
booktitle = {Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology},
articleno = {6},
numpages = {17},
keywords = {generative model, painting interactions, text-to-image generation},
location = {San Francisco, CA, USA},
series = {UIST '23}
}

@inproceedings{chung2025,
author = {Chung, John Joon Young and Roemmele, Melissa and Kreminski, Max},
title = {Toyteller: AI-powered Visual Storytelling Through Toy-Playing with Character Symbols},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3713435},
doi = {10.1145/3706598.3713435},
abstract = {We introduce Toyteller, an AI-powered storytelling system where users generate a mix of story text and visuals by directly manipulating character symbols like they are toy-playing. Anthropomorphized symbol motions can convey rich and nuanced social interactions; Toyteller leverages these motions (1) to let users steer story text generation and (2) as a visual output format that accompanies story text. We enabled motion-steered text generation and text-steered motion generation by mapping motions and text onto a shared semantic space so that large language models and motion generation models can use it as a translational layer. Technical evaluations showed that Toyteller outperforms a competitive baseline, GPT-4o. Our user study identified that toy-playing helps express intentions difficult to verbalize. However, only motions could not express all user intentions, suggesting combining it with other modalities like language. We discuss the design space of toy-playing interactions and implications for technical HCI research on human-AI interaction.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {331},
numpages = {23},
keywords = {visual storytelling, toy-playing, generative AI},
location = {
},
series = {CHI '25}
}

@inproceedings{dang2023,
author = {Dang, Hai and Brudy, Frederik and Fitzmaurice, George and Anderson, Fraser},
title = {WorldSmith: Iterative and Expressive Prompting for World Building with a Generative AI},
year = {2023},
isbn = {9798400701320},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3586183.3606772},
doi = {10.1145/3586183.3606772},
abstract = {Crafting a rich and unique environment is crucial for fictional world-building, but can be difficult to achieve since illustrating a world from scratch requires time and significant skill. We investigate the use of recent multi-modal image generation systems to enable users iteratively visualize and modify elements of their fictional world using a combination of text input, sketching, and region-based filling. WorldSmith enables novice world builders to quickly visualize a fictional world with layered edits and hierarchical compositions. Through a formative study (4 participants) and first-use study (13 participants) we demonstrate that WorldSmith offers more expressive interactions with prompt-based models. With this work, we explore how creatives can be empowered to leverage prompt-based generative AI as a tool in their creative process, beyond current "click-once" prompting UI paradigms.},
booktitle = {Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology},
articleno = {63},
numpages = {17},
keywords = {AI-assisted creativity, Fictional world-building, Multi-modal image generation},
location = {San Francisco, CA, USA},
series = {UIST '23}
}

@inproceedings{duan2025,
author = {Duan, Runlin and Zhu, Chenfei and Chen, Yuzhao and Hu, Yichen and Shi, Jingyu and Ramani, Karthik},
title = {DesignFromX:Empowering Consumer-Driven Design Space Exploration through Feature Composition of Referenced Products},
year = {2025},
isbn = {9798400714856},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3715336.3735824},
doi = {10.1145/3715336.3735824},
abstract = {Industrial products are designed to satisfy the needs of consumers. The rise of generative artificial intelligence (GenAI) enables consumers to easily modify a product by prompting a generative model, opening up opportunities to incorporate consumers in exploring the product design space. However, consumers often struggle to articulate their preferred product features due to their unfamiliarity with terminology and their limited understanding of the structure of product features. We present DesignFromX, a system that empowers consumer-driven design space exploration by helping consumers to design a product based on their preferences. Leveraging an effective GenAI-based framework, the system allows users to easily identify design features from product images and compose those features to generate conceptual images and 3D models of a new product. A user study with 24 participants demonstrates that DesignFromX lowers the barriers and frustration for consumer-driven design space explorations by enhancing both engagement and enjoyment for the participants.},
booktitle = {Proceedings of the 2025 ACM Designing Interactive Systems Conference},
pages = {1040–1060},
numpages = {21},
keywords = {User Interface Design, Generative AI, Product Design},
location = {
},
series = {DIS '25}
}

@inproceedings{elsharif2023,
author = {Elsharif, Wala and She, James and Nakov, Preslav and Wong, Simon},
title = {Enhancing Arabic Content Generation with Prompt Augmentation Using Integrated GPT and Text-to-Image Models},
year = {2023},
isbn = {9798400700286},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3573381.3596466},
doi = {10.1145/3573381.3596466},
abstract = {With the current and continuous advancements in the field of text-to-image modeling, it has become critical to design prompts that make the best of these model capabilities and guides them to generate the most desirable images, and thus the field of prompt engineering has emerged. Here, we study a method to use prompt engineering to enhance text-to-image model representation of the Arabic culture. This work proposes a simple, novel approach for prompt engineering that uses the domain knowledge of a state-of-the-art language model, GPT, to perform the task of prompt augmentation, where a simple, initial prompt is used to generate multiple, more detailed prompts related to the Arabic culture from multiple categories through a GPT model through a process known as in-context learning. The augmented prompts are then used to generate images enhanced for the Arabic culture. We perform multiple experiments with a number of participants to evaluate the performance of the proposed method, which shows promising results, specially for generating prompts that are more inclusive of the different Arabic countries and with a wider variety in terms of image subjects, where we find that our proposed method generates image with more variety 85 % of the time and are more inclusive of the Arabic countries more than 72.66 % of the time, compared to the direct approach.},
booktitle = {Proceedings of the 2023 ACM International Conference on Interactive Media Experiences},
pages = {276–288},
numpages = {13},
keywords = {Prompt engineering, Integrated systems, GPT, Arabic culture},
location = {Nantes, France},
series = {IMX '23}
}

@inproceedings{fong2024,
author = {Fong, Bi Qi and See, John},
title = {BrandDiffusion: Multimodal Personalized Marketing Visual Content Generation},
year = {2024},
isbn = {9798400711947},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3688867.3690175},
doi = {10.1145/3688867.3690175},
abstract = {Creating visual content such as product advertisements for marketing purposes has attracted research attention recently. Traditionally, such visuals showcase the product against a specific backdrop while adhering to a consistent corporate style to maintain brand visual identity. We present BrandDiffusion, a marketing image generation framework that leverages the power of pre-trained Stable Diffusion models while allowing designers to generate ideas by adapting prompt-controllable models to capture the intended marketing style. In addition, it is also capable of automatic product placement which incorporates harmonization of the product image with the generated background, guided by saliency detection, BLIP captioning, and SDEdit denoising process. We showcase the capabilities of the proposed framework through quantitative and qualitative evaluations, highlighting the quality of the generated content, its relevancy to user prompts, and a strong brand coherency.},
booktitle = {Proceedings of the 2nd International Workshop on Multimedia Content Generation and Evaluation: New Methods and Practice},
pages = {72–77},
numpages = {6},
keywords = {diffusion models, image generation, image harmonization, stable diffusion, visual style},
location = {Melbourne VIC, Australia},
series = {McGE '24}
}

@article{gu2024,
author = {Gu, Zheng and Yang, Shiyuan and Liao, Jing and Huo, Jing and Gao, Yang},
title = {Analogist: Out-of-the-box Visual In-Context Learning with Image Diffusion Model},
year = {2024},
issue_date = {July 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {43},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3658136},
doi = {10.1145/3658136},
abstract = {Visual In-Context Learning (ICL) has emerged as a promising research area due to its capability to accomplish various tasks with limited example pairs through analogical reasoning. However, training-based visual ICL has limitations in its ability to generalize to unseen tasks and requires the collection of a diverse task dataset. On the other hand, existing methods in the inference-based visual ICL category solely rely on textual prompts, which fail to capture fine-grained contextual information from given examples and can be time-consuming when converting from images to text prompts. To address these challenges, we propose Analogist, a novel inference-based visual ICL approach that exploits both visual and textual prompting techniques using a text-to-image diffusion model pretrained for image inpainting. For visual prompting, we propose a self-attention cloning (SAC) method to guide the fine-grained structural-level analogy between image examples. For textual prompting, we leverage GPT-4V's visual reasoning capability to efficiently generate text prompts and introduce a cross-attention masking (CAM) operation to enhance the accuracy of semantic-level analogy guided by text prompts. Our method is out-of-the-box and does not require fine-tuning or optimization. It is also generic and flexible, enabling a wide range of visual tasks to be performed in an in-context manner. Extensive experiments demonstrate the superiority of our method over existing approaches, both qualitatively and quantitatively. Our project webpage is available at https://analogist2d.github.io.},
journal = {ACM Trans. Graph.},
month = jul,
articleno = {130},
numpages = {15},
keywords = {visual in-context learning, diffusion models, image transformation}
}

@inproceedings{gunturu2025,
author = {Gunturu, Aditya and Pearman, Ben and Ihara, Keiichi and Faraji, Morteza and Wang, Bryan and Kazi, Rubaiat Habib and Suzuki, Ryo},
title = {MapStory: Prototyping Editable Map Animations with LLM Agents},
year = {2025},
isbn = {9798400720376},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3746059.3747664},
doi = {10.1145/3746059.3747664},
abstract = {We introduce MapStory, an LLM‑powered animation prototyping tool that generates editable map animation sequences directly from natural language text by leveraging a dual-agent LLM architecture. Given a user-written script, MapStory automatically produces a scene breakdown, which decomposes the text into key map animation primitives such as camera movements, visual highlights, and animated elements. Our system includes a researcher agent that accurately queries geospatial information by leveraging an LLM with web search, enabling automatic extraction of relevant regions, paths, and coordinates while allowing users to edit and query for changes or additional information to refine the results. Additionally, users can fine-tune parameters of these primitive blocks through an interactive timeline editor. We detail the system’s design and architecture, informed by formative interviews with professional animators and by an analysis of 200 existing map animation videos. Our evaluation, which includes expert interviews (N=5), and a usability study (N=12), demonstrates that MapStory enables users to create map animations with ease, facilitates faster iteration, encourages creative exploration, and lowers barriers to creating map-centric stories.},
booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology},
articleno = {8},
numpages = {20},
keywords = {text-to-animation, map-based storytelling, LLM-based authoring tools, AI-assisted animation, human-AI collaboration},
location = {
},
series = {UIST '25}
}

@inproceedings{hammad2025,
author = {Hammad, Noor and Fraser, C. Ailie and Harpstead, Erik and Hammer, Jessica and Dontcheva, Mira},
title = {“It’s more of a vibe I’m going for”: Designing Text-to-Music Generation Interfaces for Video Creators},
year = {2025},
isbn = {9798400714856},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3715336.3735814},
doi = {10.1145/3715336.3735814},
abstract = {Background music plays a crucial role in social media videos, yet finding the right music remains a challenge for video creators. These creators, often not music experts, struggle to describe their musical goals and compare options. AI text-to-music generation presents an opportunity to address these challenges by allowing users to generate music through text prompts; however, these models often require musical expertise and are difficult to control. In this paper, we explore how to incorporate music generation into video editing workflows. A formative study with video creators revealed challenges in articulating and iterating on musical preferences, as creators described music as “vibes” rather than with explicit musical vocabulary. Guided by these insights, we developed a creative assistant for music generation using editable vibe-based recommendations and structured refinement of music output. A user study showed that the assistant supports exploration, while direct prompting is more effective for precise goals. Our findings offer design recommendations for AI music tools for video creators.},
booktitle = {Proceedings of the 2025 ACM Designing Interactive Systems Conference},
pages = {2738–2754},
numpages = {17},
keywords = {Music Generation, Video Editing, Generative AI},
location = {
},
series = {DIS '25}
}

@inproceedings{han2025,
author = {Han, Evans Xu and Zhang, Alice Qian and Zhu, Haiyi and Shen, Hong and Liang, Paul Pu and Hsieh, Jane},
title = {POET: Supporting Prompting Creativity and Personalization with Automated Expansion of Text-to-Image Generation},
year = {2025},
isbn = {9798400720376},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3746059.3747710},
doi = {10.1145/3746059.3747710},
abstract = {State-of-the-art visual generative AI tools hold immense potential to assist users in the early ideation stages of creative tasks — offering the ability to generate (rather than search for) novel and unprecedented (instead of existing) images of considerable quality that also adhere to boundless combinations of user specifications. However, many large-scale text-to-image systems are designed for broad applicability, yielding conventional output that may limit creative exploration. They also employ interaction methods that may be difficult for beginners. Given that creative end-users often operate in diverse, context-specific ways that are often unpredictable, more variation and personalization are necessary. We introduce POET, a real-time interactive tool that (1) automatically discovers dimensions of homogeneity in text-to-image generative models, (2) expands these dimensions to diversify the output space of generated images, and (3) learns from user feedback to personalize expansions. An evaluation with 28 users spanning four creative task domains demonstrated POET’s ability to generate results with higher perceived diversity and help users reach satisfaction in fewer prompts during creative tasks, thereby prompting them to deliberate and reflect more on a wider range of possible produced results during the co-creative process. Focusing on visual creativity, POET offers a first glimpse of how interaction techniques of future text-to-image generation tools may support and align with more pluralistic values and the needs of end-users during the ideation stages of their work1.},
booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology},
articleno = {162},
numpages = {18},
keywords = {Text-to-image generation, creativity, personalization, pluralism},
location = {
},
series = {UIST '25}
}

@inproceedings{hassan2022,
author = {Hassan, Saad and Amin, Akhter Al and Gordon, Alexis and Lee, Sooyeon and Huenerfauth, Matt},
title = {Design and Evaluation of Hybrid Search for American Sign Language to English Dictionaries: Making the Most of Imperfect Sign Recognition},
year = {2022},
isbn = {9781450391573},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3491102.3501986},
doi = {10.1145/3491102.3501986},
abstract = {Searching for the meaning of an unfamiliar sign-language word in a dictionary is difficult for learners, but emerging sign-recognition technology will soon enable users to search by submitting a video of themselves performing the word they recall. However, sign-recognition technology is imperfect, and users may need to search through a long list of possible results when seeking a desired result. To speed this search, we present a hybrid-search approach, in which users begin with a video-based query and then filter the search results by linguistic properties, e.g., handshape. We interviewed 32 ASL learners about their preferences for the content and appearance of the search-results page and filtering criteria. A between-subjects experiment with 20 ASL learners revealed that our hybrid search system outperformed a video-based search system along multiple satisfaction and performance metrics. Our findings provide guidance for designers of video-based sign-language dictionary search systems, with implications for other search scenarios.},
booktitle = {Proceedings of the 2022 CHI Conference on Human Factors in Computing Systems},
articleno = {195},
numpages = {13},
keywords = {American Sign Language (ASL), Dictionary, IR Effectiveness, Search Evaluation, Search Interfaces, Search System Design, Sign Languages, User Satisfaction, Video Search},
location = {New Orleans, LA, USA},
series = {CHI '22}
}

@inproceedings{he2023,
author = {He, Huiguo and Wang, Tianfu and Yang, Huan and Fu, Jianlong and Yuan, Nicholas Jing and Yin, Jian and Chao, Hongyang and Zhang, Qi},
title = {Learning Profitable NFT Image Diffusions via Multiple Visual-Policy Guided Reinforcement Learning},
year = {2023},
isbn = {9798400701085},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3581783.3612595},
doi = {10.1145/3581783.3612595},
abstract = {We study the task of generating profitable Non-Fungible Token (NFT) images from user-input texts. Recent advances in diffusion models have shown great potential for image generation. However, existing works can fall short in generating visually-pleasing and highly-profitable NFT images, mainly due to the lack of 1) plentiful and fine-grained visual attribute prompts for an NFT image, and 2) effective optimization metrics for generating high-quality NFT images. To solve these challenges, we propose a Diffusion based generation framework with Multiple Visual-Policies as rewards (i.e., Diffusion-MVP) for NFT images. The proposed framework consists of a large language model (LLM), a diffusion-based image generator, and a series of visual rewards by design. First, the LLM enhances a basic human input (such as "panda") by generating more comprehensive NFT-style prompts that include specific visual attributes, such as "panda with Ninja style and green background." Second, the diffusion-based image generator is fine-tuned using a large-scale NFT dataset to capture fine-grained image styles and accessory compositions of popular NFT elements. Third, we further propose to utilize multiple visual-policies as optimization goals, including visual rarity levels, visual aesthetic scores, and CLIP-based text-image relevances. This design ensures that our proposed Diffusion-MVP is capable of minting NFT images with high visual quality and market value. To facilitate this research, we have collected the largest publicly available NFT image dataset to date, consisting of 1.5 million high-quality images with corresponding texts and market values. Extensive experiments including objective evaluations and user studies demonstrate that our framework can generate NFT images showing more visually engaging elements and higher market value, compared with state-of-the-art approaches.},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {6831–6840},
numpages = {10},
keywords = {diffusion model, image generation, nft, policy learning},
location = {Ottawa ON, Canada},
series = {MM '23}
}

@inproceedings{huh2023,
author = {Huh, Mina and Peng, Yi-Hao and Pavel, Amy},
title = {GenAssist: Making Image Generation Accessible},
year = {2023},
isbn = {9798400701320},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3586183.3606735},
doi = {10.1145/3586183.3606735},
abstract = {Blind and low vision (BLV) creators use images to communicate with sighted audiences. However, creating or retrieving images is challenging for BLV creators as it is difficult to use authoring tools or assess image search results. Thus, creators limit the types of images they create or recruit sighted collaborators. While text-to-image generation models let creators generate high-fidelity images based on a text description (i.e. prompt), it is difficult to assess the content and quality of generated images. We present GenAssist, a system to make text-to-image generation accessible. Using our interface, creators can verify whether generated image candidates followed the prompt, access additional details in the image not specified in the prompt, and skim a summary of similarities and differences between image candidates. To power the interface, GenAssist uses a large language model to generate visual questions, vision-language models to extract answers, and a large language model to summarize the results. Our study with 12 BLV creators demonstrated that GenAssist enables and simplifies the process of image selection and generation, making visual authoring more accessible to all.},
booktitle = {Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology},
articleno = {38},
numpages = {17},
keywords = {Accessibility, Creativity Support Tools, Generative AI, Image Generation},
location = {San Francisco, CA, USA},
series = {UIST '23}
}

@article{jiang2024,
author = {Jiang, Jianan and Wu, Di and Deng, Hanhui and Long, Yidan and Tang, Wenyi and Li, Xiang and Liu, Can and Jin, Zhanpeng and Zhang, Wenlei and Qi, Tangquan},
title = {HAIGEN: Towards Human-AI Collaboration for Facilitating Creativity and Style Generation in Fashion Design},
year = {2024},
issue_date = {September 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {8},
number = {3},
url = {https://doi.org/10.1145/3678518},
doi = {10.1145/3678518},
abstract = {The process of fashion design usually involves sketching, refining, and coloring, with designers drawing inspiration from various images to fuel their creative endeavors. However, conventional image search methods often yield irrelevant results, impeding the design process. Moreover, creating and coloring sketches can be time-consuming and demanding, acting as a bottleneck in the design workflow. In this work, we introduce HAIGEN (Human-AI Collaboration for GENeration), an efficient fashion design system for Human-AI collaboration developed to aid designers. Specifically, HAIGEN consists of four modules. T2IM, located in the cloud, generates reference inspiration images directly from text prompts. With three other modules situated locally, the I2SM batch generates the image material library into a certain designer-style sketch material library. The SRM recommends similar sketches in the generated library to designers for further refinement, and the STM colors the refined sketch according to the styles of inspiration images. Through our system, any designer can perform local personalized fine-tuning and leverage the powerful generation capabilities of large models in the cloud, streamlining the entire design development process. Given that our approach integrates both cloud and local model deployment schemes, it effectively safeguards design privacy by avoiding the need to upload personalized data from local designers. We validated the effectiveness of each module through extensive qualitative and quantitative experiments. User surveys also confirmed that HAIGEN offers significant advantages in design efficiency, positioning it as a new generation of aid-tool for designers.},
journal = {Proc. ACM Interact. Mob. Wearable Ubiquitous Technol.},
month = sep,
articleno = {107},
numpages = {27},
keywords = {Generative Artificial Intelligence, Human-AI Collaboration, Personalized Fashion Design}
}

@inproceedings{kamath2024,
author = {Kamath, Purnima and Morreale, Fabio and Bagaskara, Priambudi Lintang and Wei, Yize and Nanayakkara, Suranga},
title = {Sound Designer-Generative AI Interactions: Towards Designing Creative Support Tools for Professional Sound Designers},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642040},
doi = {10.1145/3613904.3642040},
abstract = {The practice of sound design involves creating and manipulating environmental sounds for music, films, or games. Recently, an increasing number of studies have adopted generative AI to assist in sound design co-creation. Most of these studies focus on the needs of novices, and less on the pragmatic needs of sound design practitioners. In this paper, we aim to understand how generative AI models might support sound designers in their practice. We designed two interactive generative AI models as Creative Support Tools (CSTs) and invited nine professional sound design practitioners to apply the CSTs in their practice. We conducted semi-structured interviews and reflected on the challenges and opportunities of using generative AI in mixed-initiative interfaces for sound design. We provide insights into sound designers’ expectations of generative AI and highlight opportunities to situate generative AI-based tools within the design process. Finally, we discuss design considerations for human-AI interaction researchers working with audio.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {730},
numpages = {17},
keywords = {Audio, Creative Support Tools, Generative AI, Mixed-Initiative Creative Interfaces, Sound design},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{kim2025,
author = {Kim, Hui-Jun and Kim, Jeongho and Jeong, Sohyun and Lee, Minbong and Choo, Jaegul and Kim, Sung-Hee},
title = {ShoeGenAI: A Creativity Support Tool Bridging Design Intention and Feasibility in Shoe Design},
year = {2025},
isbn = {9798400720376},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3746059.3747691},
doi = {10.1145/3746059.3747691},
abstract = {Product designers increasingly turn to generative AI for creating concept images, but these outputs often fall short in terms of real-world manufacturability and typically require iterative revisions to align with intended designs. Concentrating on sneaker design, we introduce ShoeGenAI, an AI tool that enhances designers’ creativity while ensuring feasible outcomes and reducing the need for post-processing. A formative study involving four shoe designers uncovered key limitations in both traditional workflows and current genereative AI tools. These insights guided the development of four core features: fine-tuned models trained on domain-specific data, template-driven prompt assistance, support for hybrid part recombination, and localized editing for detail refinement. A subsequent user study with 20 designers showed that ShoeGenAI enabled clearer communication of design intent, more efficient workflows with less manual correction, and higher satisfaction with the realism and feasibility of the generated outputs. We also explore how professionals and novices differ in their use of creativity support tools, especially across tasks ranging from imitation to original creation.},
booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology},
articleno = {6},
numpages = {18},
keywords = {High-feasibility, Generative Model, Creativity Support, Fashion/Clothing, Prototyping/Implementation},
location = {
},
series = {UIST '25}
}

@inproceedings{kumari2024,
author = {Kumari, Nupur and Su, Grace and Zhang, Richard and Park, Taesung and Shechtman, Eli and Zhu, Jun-Yan},
title = {Customizing Text-to-Image Diffusion with Object Viewpoint Control},
year = {2024},
isbn = {9798400711312},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3680528.3687564},
doi = {10.1145/3680528.3687564},
abstract = {Model customization introduces new concepts to existing text-to-image models, enabling the generation of these new concepts/objects in novel contexts. However, such methods lack accurate camera view control with respect to the new object, and users must resort to prompt engineering (e.g., adding “top-view”) to achieve coarse view control. In this work, we introduce a new task – enabling explicit control of the object viewpoint in the customization of text-to-image diffusion models. This allows us to modify the custom object’s properties and generate it in various background scenes via text prompts, all while incorporating the object viewpoint as an additional control. This new task presents significant challenges, as one must harmoniously merge a 3D representation from the multi-view images with the 2D pre-trained model. To bridge this gap, we propose to condition the diffusion process on the 3D object features rendered from the target viewpoint. During training, we fine-tune the 3D feature prediction modules to reconstruct the object’s appearance and geometry, while reducing overfitting to the input multi-view images. Our method outperforms existing image editing and model customization baselines in preserving the custom object’s identity while following the target object viewpoint and the text prompt.},
booktitle = {SIGGRAPH Asia 2024 Conference Papers},
articleno = {7},
numpages = {13},
keywords = {Image Generation, Diffusion Models, Deep Generative Models, Model Customization},
location = {Tokyo, Japan},
series = {SA '24}
}

@inproceedings{kun2024,
author = {Kun, Peter and Freiberger, Matthias and Risi, Sebastian and L\o{}vlie, Anders Sundnes},
title = {GenFrame: An Interactive Picture Frame Painting the Portrait of Any Girl},
year = {2024},
isbn = {9798400706325},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3656156.3665437},
doi = {10.1145/3656156.3665437},
abstract = {GenFrame is an interactive picture frame that mimics traditional paintings while being equipped with generative AI capabilities. Painting “portraits of a girl” has been a trope in art history for centuries. However, in the era of generative AI models, the permanence of traditional art is challenged. When an AI model is able to mimic any painting style, it can also give agency to the viewer to repaint the painting per individual desires. In this installation, we modify the role of the museum placard to provide a minimal tangible interface to change the style of the image and the mood of the depicted girl. In this way, the generative AI model is interfaced in tangible ways, instead of the regular prompt paradigm. Demo video available here: https://youtu.be/ga7pFgAOPiY},
booktitle = {Companion Publication of the 2024 ACM Designing Interactive Systems Conference},
pages = {353–358},
numpages = {6},
keywords = {AI art, AI-generated painting, generative AI, image generation, interactive art},
location = {IT University of Copenhagen, Denmark},
series = {DIS '24 Companion}
}

@inproceedings{lin2025,
author = {Lin, Haichuan and Ye, Yilin and Xia, Jiazhi and Zeng, Wei},
title = {SketchFlex: Facilitating Spatial-Semantic Coherence in Text-to-Image Generation with Region-Based Sketches},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3713801},
doi = {10.1145/3706598.3713801},
abstract = {Text-to-image models can generate visually appealing images from text descriptions. Efforts have been devoted to improving model controls with prompt tuning and spatial conditioning. However, our formative study highlights the challenges for non-expert users in crafting appropriate prompts and specifying fine-grained spatial conditions (e.g., depth or canny references) to generate semantically cohesive images, especially when multiple objects are involved. In response, we introduce SketchFlex, an interactive system designed to improve the flexibility of spatially conditioned image generation using rough region sketches. The system automatically infers user prompts with rational descriptions within a semantic space enriched by crowd-sourced object attributes and relationships. Additionally, SketchFlex refines users’ rough sketches into canny-based shape anchors, ensuring the generation quality and alignment of user intentions. Experimental results demonstrate that SketchFlex achieves more cohesive image generations than end-to-end models, meanwhile significantly reducing cognitive load and better matching user intentions compared to region-based generation baseline.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {546},
numpages = {19},
keywords = {Generative artificial intelligence, Diffusion model},
location = {
},
series = {CHI '25}
}

@inproceedings{liu2022,
author = {Liu, Vivian and Qiao, Han and Chilton, Lydia},
title = {Opal: Multimodal Image Generation for News Illustration},
year = {2022},
isbn = {9781450393201},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3526113.3545621},
doi = {10.1145/3526113.3545621},
abstract = {Advances in multimodal AI have presented people with powerful ways to create images from text. Recent work has shown that text-to-image generations are able to represent a broad range of subjects and artistic styles. However, finding the right visual language for text prompts is difficult. In this paper, we address this challenge with Opal, a system that produces text-to-image generations for news illustration. Given an article, Opal guides users through a structured search for visual concepts and provides a pipeline allowing users to generate illustrations based on an article’s tone, keywords, and related artistic styles. Our evaluation shows that Opal efficiently generates diverse sets of news illustrations, visual assets, and concept ideas. Users with Opal generated two times more usable results than users without. We discuss how structured exploration can help users better understand the capabilities of human AI co-creative systems.},
booktitle = {Proceedings of the 35th Annual ACM Symposium on User Interface Software and Technology},
articleno = {73},
numpages = {17},
keywords = {text-to-image generation, prompt engineering, news illustration, multimodal, ideation, creativity support tools, co-creativity, applied AI},
location = {Bend, OR, USA},
series = {UIST '22}
}

@inproceedings{liu2023,
author = {Liu, Vivian and Vermeulen, Jo and Fitzmaurice, George and Matejka, Justin},
title = {3DALL-E: Integrating Text-to-Image AI in 3D Design Workflows},
year = {2023},
isbn = {9781450398930},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3563657.3596098},
doi = {10.1145/3563657.3596098},
abstract = {Text-to-image AI are capable of generating novel images for inspiration, but their applications for 3D design workflows and how designers can build 3D models using AI-provided inspiration have not yet been explored. To investigate this, we integrated DALL-E, GPT-3, and CLIP within a CAD software in 3DALL-E, a plugin that generates 2D image inspiration for 3D design. 3DALL-E allows users to construct text and image prompts based on what they are modeling. In a study with 13 designers, we found that designers saw great potential in 3DALL-E within their workflows and could use text-to-image AI to produce reference images, prevent design fixation, and inspire design considerations. We elaborate on prompting patterns observed across 3D modeling tasks and provide measures of prompt complexity observed across participants. From our findings, we discuss how 3DALL-E can merge with existing generative design workflows and propose prompt bibliographies as a form of human-AI design history.},
booktitle = {Proceedings of the 2023 ACM Designing Interactive Systems Conference},
pages = {1955–1977},
numpages = {23},
keywords = {3D design, 3D modeling, AI applications, CAD, CLIP, DALL-E, GPT-3, co-creativity, creative copilot, creativity support tools, diffusion, ideation, multimodal, prompt engineering, text-to-3D, text-to-image, workflow},
location = {Pittsburgh, PA, USA},
series = {DIS '23}
}

@inproceedings{manesh2024,
author = {Aghel Manesh, Setareh and Zhang, Tianyi and Onishi, Yuki and Hara, Kotaro and Bateman, Scott and Li, Jiannan and Tang, Anthony},
title = {How People Prompt Generative AI to Create Interactive VR Scenes},
year = {2024},
isbn = {9798400705830},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3643834.3661547},
doi = {10.1145/3643834.3661547},
abstract = {Generative AI tools can provide people with the ability to create virtual environments and scenes with natural language prompts. Yet, how people will formulate such prompts is unclear—particularly when they inhabit the environment that they are designing. For instance, it is likely that a person might say, “Put a chair here,” while pointing at a location. If such linguistic and embodied features are common to people’s prompts, we need to tune models to accommodate them. In this work, we present a Wizard of Oz elicitation study with 22 participants, where we studied people’s implicit expectations when verbally prompting such programming agents to create interactive VR scenes. Our findings show when people prompted the agent, they had several implicit expectations of these agents: (1) they should have an embodied knowledge of the environment; (2) they should understand embodied prompts by users; (3) they should recall previous states of the scene and the conversation, and that (4) they should have a commonsense understanding of objects in the scene. Further, we found that participants prompted differently when they were prompting in situ (i.e. within the VR environment) versus ex situ (i.e. viewing the VR environment from the outside). To explore how these lessons could be applied, we designed and built Ostaad, a conversational programming agent that allows non-programmers to design interactive VR experiences that they inhabit. Based on these explorations, we outline new opportunities and challenges for conversational programming agents that create VR environments.},
booktitle = {Proceedings of the 2024 ACM Designing Interactive Systems Conference},
pages = {2319–2340},
numpages = {22},
keywords = {embodied interaction, embodied prompting, generative ai, interactive virtual reality, multi-modal, prompting, virtual reality},
location = {Copenhagen, Denmark},
series = {DIS '24}
}

@inproceedings{mao2024,
author = {Mao, Qi and Chen, Lan and Gu, Yuchao and Fang, Zhen and Shou, Mike Zheng},
title = {MAG-Edit: Localized Image Editing in Complex Scenarios via Mask-Based Attention-Adjusted Guidance},
year = {2024},
isbn = {9798400706868},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3664647.3680830},
doi = {10.1145/3664647.3680830},
abstract = {Recent diffusion-based image editing approaches have exhibited impressive editing capabilities in images with one dominant object in simple compositions. However, localized editing in images containing multiple objects and intricate compositions has not been well-studied in the literature, despite its growing real-world demands. Existing mask-based inpainting methods fall short of retaining the underlying structure within the edit region, causing noticeable discordance with their complex surroundings. Meanwhile, attention-based methods such as Prompt-to-Prompt (P2P) often exhibit editing leakage and misalignment in more complex compositions. In this work, we propose MAG-Edit, a plug-and-play, inference-stage optimization method, that empowers attention-based editing approaches, such as P2P, to enhance localized image editing in intricate scenarios. In particular, MAG-Edit optimizes the noise latent feature by encouraging two mask-based cross-attention ratios of the edit token, which in turn gradually enhances the local alignment with the desired prompt. Extensive quantitative and qualitative experiments demonstrate the effectiveness of our method in achieving both text alignment and structure preservation for localized editing within complex scenarios.},
booktitle = {Proceedings of the 32nd ACM International Conference on Multimedia},
pages = {6842–6850},
numpages = {9},
keywords = {attention-based guidance, diffusion models, text-based image editing},
location = {Melbourne VIC, Australia},
series = {MM '24}
}

@inproceedings{marquardt2025,
author = {Marquardt, Nicolai and Roseway, Asta and Romat, Hugo and Panda, Payod and Pahud, Michel and Ramos, Gonzalo and Drucker, Steven M. and Wilson, Andrew D. and Hinckley, Ken and Riche, Nathalie},
title = {ImaginationVellum: Generative-AI Ideation Canvas with Spatial Prompts, Generative Strokes, and Ideation History},
year = {2025},
isbn = {9798400720376},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3746059.3747631},
doi = {10.1145/3746059.3747631},
abstract = {We introduce ImaginationVellum, a multi-modal spatial canvas for early-stage visual ideation and concept sketching with generative AI. The resulting system supports a unique style of human-AI co-creation where  the canvas is the prompt. This means that ImaginationVellum employs the entire 2D canvas as an active prompt space, where spatial arrangement, proximity, and composition of diverse content elements—inking, text, images, and intermediate results—steer generative visual outcomes. As a technical probe, ImaginationVellum contributes a set of spatially-grounded direct manipulation tools for iterative visual ideation. In particular, we introduce Generative Strokes—freeform strokes that spatially modulate generation and prompt-parameters (articulated along multiple latent semantic or stylistic dimensions). These techniques afford rapid traversal of design spaces via convergence, divergence, re-composition, blending, and remixing of concepts. We detail the system architecture, design rationale, proximity-dependent intent tags for localized control, and methods for spatial prompting and varying output along spatial gradients. Temporal replay and visualization of provenance make ideation trajectories actionable, turning the design process itself into an artifact that supports reflection-in-action and revisitation of design decisions. We report insights from a preliminary study of how users construct, steer, and revisit ideas using spatial prompts, and discuss tradeoffs in modulating spatially-dependent content generation.},
booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology},
articleno = {159},
numpages = {19},
keywords = {spatial prompting, generative strokes, generative AI, human-AI co-creation, sketching, spatial ideation canvas},
location = {
},
series = {UIST '25}
}

@inproceedings{masson2024,
author = {Masson, Damien and Malacria, Sylvain and Casiez, G\'{e}ry and Vogel, Daniel},
title = {DirectGPT: A Direct Manipulation Interface to Interact with Large Language Models},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642462},
doi = {10.1145/3613904.3642462},
abstract = {We characterize and demonstrate how the principles of direct manipulation can improve interaction with large language models. This includes: continuous representation of generated objects of interest; reuse of prompt syntax in a toolbar of commands; manipulable outputs to compose or control the effect of prompts; and undo mechanisms. This idea is exemplified in DirectGPT, a user interface layer on top of ChatGPT that works by transforming direct manipulation actions to engineered prompts. A study shows participants were 50% faster and relied on 50% fewer and 72% shorter prompts to edit text, code, and vector images compared to baseline ChatGPT. Our work contributes a validated approach to integrate LLMs into traditional software using direct manipulation. Data, code, and demo available at https://osf.io/3wt6s.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {975},
numpages = {16},
keywords = {direct manipulation, large language models, prompt engineering},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{michelessa2025,
author = {Michelessa, Mario and Ng, Jamie and Hurter, Christophe and Lim, Brian Y.},
title = {Varif.ai to Vary and Verify User-Driven Diversity in Scalable Image Generation},
year = {2025},
isbn = {9798400714856},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3715336.3735847},
doi = {10.1145/3715336.3735847},
abstract = {Diversity in image generation is essential to ensure fair representations and support creativity in ideation. Hence, many text-to-image models have implemented diversification mechanisms. Yet, after a few iterations of generation, a lack of diversity becomes apparent, because each user has their own diversity goals (e.g., different colors, brands of cars), and there are diverse attributions to be specified. To support user-driven diversity control, we propose Varif.ai that employs text-to-image and Large Language Models to iteratively i) (re)generate a set of images, ii) verify if user-specified attributes have sufficient coverage, and iii) vary existing or new attributes. Through an elicitation study, we uncovered user needs for diversity in image generation. A pilot validation showed that Varif.ai made achieving diverse image sets easier. In a controlled evaluation with 20 participants, Varif.ai proved more effective than baseline methods across various scenarios. Thus, this supports user control of diversity in image generation for creative ideation and scalable image generation.},
booktitle = {Proceedings of the 2025 ACM Designing Interactive Systems Conference},
pages = {1867–1885},
numpages = {19},
keywords = {Image generation, human-AI interaction, diversity, creativity tools},
location = {
},
series = {DIS '25}
}

@inproceedings{mohian2022,
author = {Mohian, Soumik and Csallner, Christoph},
title = {PSDoodle: fast app screen search via partial screen doodle},
year = {2022},
isbn = {9781450393010},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3524613.3527816},
doi = {10.1145/3524613.3527816},
abstract = {Searching through existing repositories for a specific mobile app screen design is currently either slow or tedious. Such searches are either limited to basic keyword searches (Google Image Search) or require as input a complete query screen image (SWIRE). A promising alternative is interactive partial sketching, which is more structured than keyword search and faster than complete-screen queries. PSDoodle is the first system to allow interactive search of screens via interactive sketching. PSDoodle is built on top of a combination of the Rico repository of some 58k Android app screens, the Google QuickDraw dataset of icon-level doodles, and DoodleUINet, a curated corpus of some 10k app icon doodles collected from hundreds of individuals. In our evaluation with third-party software developers, PSDoodle provided similar top-10 screen retrieval accuracy as the state of the art from the SWIRE line of work, while cutting the average time required about in half.},
booktitle = {Proceedings of the 9th IEEE/ACM International Conference on Mobile Software Engineering and Systems},
pages = {89–99},
numpages = {11},
keywords = {GUI, SBIR, deep learning, design examples, sketch-based image retrieval, sketching, user interface design},
location = {Pittsburgh, Pennsylvania},
series = {MOBILESoft '22}
}

@inproceedings{mohian2023,
author = {Mohian, Soumik and Tang, Tony and Trinh, Tuan and Dang, Don and Csallner, Christoph},
title = {D2S2: Drag ’n’ Drop Mobile App Screen Search},
year = {2023},
isbn = {9798400703270},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3611643.3613100},
doi = {10.1145/3611643.3613100},
abstract = {The lack of diverse UI element representations in publicly available datasets hinders the scalability of sketch-based interactive mobile search. This paper introduces D2S2, a novel approach that addresses this limitation via drag-and-drop mobile screen search, accommodating visual and text-based queries. D2S2 searches 58k Rico screens for relevant UI examples based on UI element attributes, including type, position, shape, and text. In an evaluation with 10 novice software developers D2S2 successfully retrieves target screens within the top-20 search results in 15/19 attempts within a minute. The tool offers interactive and iterative search, updating its search results each time the user modifies the search query. Interested users can freely access D2S2 (http://pixeltoapp.com/D2S2), build on D2S2 or replicate results via D2S2’s open-source implementation (https://github.com/toni-tang/D2S2), or watch D2S2’s video demonstration (https://youtu.be/fdoYiw8lAn0).},
booktitle = {Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
pages = {2177–2181},
numpages = {5},
keywords = {User interface design, design examples, information retrieval, interactive screenshot search, prototyping},
location = {San Francisco, CA, USA},
series = {ESEC/FSE 2023}
}

@article{oppenlaender2025,
author = {Oppenlaender, Jonas and Johnston, Hannah and Silvennoinen, Johanna Maria and Barranha, Helena},
title = {Artworks Reimagined: Exploring Human-AI Co-Creation through Body Prompting},
year = {2025},
issue_date = {June 2025},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {9},
number = {4},
url = {https://doi.org/10.1145/3734189},
doi = {10.1145/3734189},
abstract = {Image generation using generative artificial intelligence has become a popular activity. However, text-to-image generation—where images are produced from typed prompts—can be less engaging in public settings since the act of typing tends to limit interactive audience participation, thereby reducing its suitability for designing dynamic public installations. In this article, we explore body prompting as input modality for image generation in the context of public event settings. Body prompting extends interaction with generative AI beyond textual inputs to reconnect the creative act of image generation with the physical act of creating artworks. We implement this concept in an interactive art installation, Artworks Reimagined, designed to transform existing artworks via body prompting. We deployed the installation at an event with hundreds of visitors in a public and private setting. Our semi-structured interviews with a sample of visitors (N = 79) show that body prompting was well-received and provides an engaging and fun experience. We present insights into participants’ experience of body prompting and AI co-creation and identify three distinct strategies of embodied interaction focused on re-creating, reimagining, or casual interaction. We provide valuable recommendations for practitioners seeking to design interactive generative AI experiences in museums, galleries, and public event spaces.},
journal = {Proc. ACM Hum.-Comput. Interact.},
month = jun,
articleno = {EICS012},
numpages = {34},
keywords = {generative AI, human-AI interaction, embodied interaction, image generation, co-creation, public displays, art installation}
}

@inproceedings{park2024,
author = {Park, Jeongeun and Shin, Hyorim and Oh, Changhoon and Kim, Ha Young},
title = {“Is Text-Based Music Search Enough to Satisfy Your Needs?” A New Way to Discover Music with Images},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642126},
doi = {10.1145/3613904.3642126},
abstract = {Music is intrinsically connected to human experience, yet the plethora of choices often renders the search for the ideal piece perplexing, especially when the search terms are ambiguous. This study questions the viability of employing visual data, specifically images, in innovative queries for music search, and it aims to better align search results with users’ moods and situational context. We designed and evaluated three prototype systems for music search—TTTune (text-based), VisTune (image-based), and VTTune (hybrid)—to comparatively assess user experience and system usability. In a comprehensive user study involving 236 participants, each participant interacted with one of the systems and subsequently completed post-experimental surveys. A subset of participants also participated in in-depth interviews to further elucidate the potential and the advantages of image-based music retrieval (IMR) systems. Our findings reveal a marked preference for the user experience and usability offered by the IMR approach, as compared with the traditional text-based method. This underscores the potential of the image in an effective search query. Based on these findings, we discuss interface design guidelines tailored for IMR systems and factors affecting system performance, contributing to the evolving landscape of music search methods.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {504},
numpages = {21},
keywords = {Image-to-music retrieval, multimodal, music search, system usability, user experience},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{pavlichenko2023,
author = {Pavlichenko, Nikita and Ustalov, Dmitry},
title = {Best Prompts for Text-to-Image Models and How to Find Them},
year = {2023},
isbn = {9781450394086},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3539618.3592000},
doi = {10.1145/3539618.3592000},
abstract = {Advancements in text-guided diffusion models have allowed for the creation of visually appealing images similar to those created by professional artists. The effectiveness of these models depends on the composition of the textual description, known as the prompt, and its accompanying keywords. Evaluating aesthetics computationally is difficult, so human input is necessary to determine the ideal prompt formulation and keyword combination. In this study, we propose a human-in-the-loop method for discovering the most effective combination of prompt keywords using a genetic algorithm. Our approach demonstrates how this can lead to an improvement in the visual appeal of images generated from the same description.},
booktitle = {Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {2067–2071},
numpages = {5},
keywords = {aesthetics evaluation, genetic optimization, human feedback, text-to-image generation},
location = {Taipei, Taiwan},
series = {SIGIR '23}
}

@inproceedings{peng2024,
author = {Peng, Xiaohan and Koch, Janin and Mackay, Wendy E.},
title = {DesignPrompt: Using Multimodal Interaction for Design Exploration with Generative AI},
year = {2024},
isbn = {9798400705830},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3643834.3661588},
doi = {10.1145/3643834.3661588},
abstract = {Visually oriented designers often struggle to create effective generative AI (GenAI) prompts. A preliminary study identified specific issues in composing and fine-tuning prompts, as well as needs in accurately translating intentions into rich input. We developed DesignPrompt, a moodboard tool that lets designers combine multiple modalities — images, color, text — into a single GenAI prompt and tweak the results. We ran a comparative structured observation study with 12 professional designers to better understand their intent expression, expectation alignment and transparency perception using DesignPrompt and text input GenAI. We found that multimodal prompt input encouraged designers to explore and express themselves more effectively. Designer’s interaction preferences change according to their overall sense of control over the GenAI and whether they are seeking inspiration or a specific image. Designers developed innovative uses of DesignPrompt, including developing elaborate multimodal prompts and creating a multimodal prompt pattern to maximize novelty while ensuring consistency.},
booktitle = {Proceedings of the 2024 ACM Designing Interactive Systems Conference},
pages = {804–818},
numpages = {15},
keywords = {Creativity Support Tool, Design Practice, Generative AI, Human-AI Ideation, Human-AI Interaction, Moodboard},
location = {Copenhagen, Denmark},
series = {DIS '24}
}

@inproceedings{peng2025,
author = {Peng, Xiaohan and Koch, Janin and Mackay, Wendy E.},
title = {FusAIn: Composing Generative AI Visual Prompts Using Pen-based Interaction},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3714027},
doi = {10.1145/3706598.3714027},
abstract = {Although current generative AI (GenAI) enables designers to create novel images, its focus on text-based and whole-image interaction limits expressive engagement with visual materials. Based on the design concept of deconstruction and reconstruction of digital visual attributes for visual prompts, we present FusAIn, a GenAI prompt composition tool that lets designers create personalized pens by loading them with objects or attributes such as color or texture. GenAI then fuses the pen’s contents to create new images. Extracting and reusing inspirational material matches designers’ existing work practices, making GenAI more contextualized for professional design. A study with 12 designers shows how FusAIn improves their ability to define visual details at different levels that are difficult to express with current GenAI prompts. Pen-based interaction lets them maintain fine-grained control over generated results, increasing GenAI image’s editability and reusability. We discuss the benefits of “composition as prompts” and directions for future research.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {874},
numpages = {20},
keywords = {Creativity Support Tools, Human-AI Interaction, Design Practice, Machine Learning, Generative AI},
location = {
},
series = {CHI '25}
}

@inproceedings{polley2022,
author = {Polley, Sayantan and Mondal, Subhajit and Mannam, Venkata Srinath and Kumar, Kushagra and Patra, Subhankar and N\"{u}rnberger, Andreas},
title = {X-Vision: Explainable Image Retrieval by Re-Ranking in Semantic Space},
year = {2022},
isbn = {9781450392365},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3511808.3557187},
doi = {10.1145/3511808.3557187},
abstract = {We present X-Vision, an explainable AI (XAI) driven image retrieval system based on a re-ranking approach to support non-expert users. We generate textual explanations such as, ''This image is similar to query image in color by Y%, shape by Z%'' along with visual explanations that compare image features. Besides the XAI goal of making AI systems transparent, we address the semantic gap between user's perception and model ranking, which arises in content based image retrieval (CBIR). We attempt to explain the notion of similarity in images in a query-by-example scenario, starting with relatively simple features such as color, texture, objects, background-foreground segments, moving to semantic representations learned from hidden layers of deep networks. The base retrieval model compares the query vector with other image feature vectors to create rankings. This result list is transferred to a semantic feature space that allows rule-based re-rankings. The core contribution of this work is a re-ranking algorithm for generating explanations. Our re-ranking improves retrieval performance (MAP) when compared with a base ranker, a random baseline, and recent CBIR baseline rankers on PASCAL VOC data. We evaluate XAI focused aspects of user trust in an eye-tracker based user study, we find that explanations supported users in the search process and understanding the notion of similarity.},
booktitle = {Proceedings of the 31st ACM International Conference on Information & Knowledge Management},
pages = {4955–4959},
numpages = {5},
keywords = {explainable ai, explainable search, xai},
location = {Atlanta, GA, USA},
series = {CIKM '22}
}

@inproceedings{rafner2023,
author = {Rafner, Janet and Zana, Blanka and Dalsgaard, Peter and Biskjaer, Michael Mose and Sherson, Jacob},
title = {Picture This: AI-Assisted Image Generation as a Resource for Problem Construction in Creative Problem-Solving},
year = {2023},
isbn = {9798400701801},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3591196.3596823},
doi = {10.1145/3591196.3596823},
abstract = {In this paper, we explore the potential of AI-assisted visualization during the problem identification and construction phase of the creative problem-solving process. We examine this within the context of the ongoing crea.visions research project, which employs AI technologies to visualize citizens' visions of the future. Our findings underscore various factors contributing to the effectiveness of assisted visualization in this setting, such as: 1) the tool's dual role as both a visual and ideational aid, 2) the introduction of innovative collaborative elements like prompt engineering, 3) the enhancement of visual expression without requiring artistic skills, and 4) the facilitation of idea communication. We also recognize limitations related to the tool and the problem context such as abstract concepts. This study serves as a foundation for future research on AI-assisted image generation as a resource in creative problem-solving, laying the groundwork for the creation of increasingly effective and user-friendly tools.},
booktitle = {Proceedings of the 15th Conference on Creativity and Cognition},
pages = {262–268},
numpages = {7},
keywords = {GANs, creative problem solving, human-AI co-creation, stable diffusion},
location = {Virtual Event, USA},
series = {C&C '23}
}

@inproceedings{riche2025,
author = {Riche, Nathalie and Offenwanger, Anna and Gmeiner, Frederic and Brown, David and Romat, Hugo and Pahud, Michel and Marquardt, Nicolai and Inkpen, Kori and Hinckley, Ken},
title = {AI-Instruments: Embodying Prompts as Instruments to Abstract & Reflect Graphical Interface Commands as General-Purpose Tools},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3714259},
doi = {10.1145/3706598.3714259},
abstract = {Chat-based prompts respond with verbose linear-sequential texts, making it difficult to explore and refine ambiguous intents, back up and reinterpret, or shift directions in creative AI-assisted design work.  AI-Instruments instead embody “prompts” as interface objects via three key principles: (1) Reification of user-intent as reusable direct-manipulation instruments; (2) Reflection of multiple interpretations of ambiguous user-intents (Reflection-in-intent) as well as the range of AI-model responses (Reflection-in-response) to inform design "moves" towards a desired result; and (3) Grounding to instantiate an instrument from an example, result, or extrapolation directly from another instrument. Further, AI-Instruments leverage LLM’s to suggest, vary, and refine new instruments, enabling a system that goes beyond hard-coded functionality by generating its own instrumental controls from content. We demonstrate four technology probes, applied to image generation, and qualitative insights from twelve participants, showing how AI-Instruments address challenges of intent formulation, steering via direct manipulation, and non-linear iterative workflows to reflect and resolve ambiguous intents.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {1104},
numpages = {18},
keywords = {instrumental interaction, generative AI interfaces},
location = {
},
series = {CHI '25}
}

@inproceedings{shaw2025,
author = {Shaw, Andrew and Ye, Andre and Krishna, Ranjay and Zhang, Amy},
title = {Agonistic Image Generation: Unsettling the Hegemony of Intention},
year = {2025},
isbn = {9798400714825},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3715275.3732030},
doi = {10.1145/3715275.3732030},
abstract = {Current image generation tools largely follow an intention-centric paradigm that aims to actualize user intentions but neglects the sociopolitical conversations that these intentions are embedded in. As these tools become cornerstones of the media landscape, however, it is increasingly evident that sociopolitical conflicts over visual representation are inescapable parts of image generation. For instance, in March 2024, Google’s Gemini faced criticism for inappropriately injecting demographic diversity into user prompts. Although Gemini challenged user intentions, its top-down imposition of a standard for “diversity” ultimately proved counterproductive. In this paper, we present an alternative approach: an image generation interface designed to embrace open negotiation along the sociopolitical dimensions of image creation. Grounded in agonistic pluralism (from the Greek agon, meaning struggle), our interface actively engages users with competing visual interpretations of their prompts. Through a lab study with 29 participants, we evaluate our agonistic interface on its ability to facilitate reflection, or engagement with other perspectives that challenges dominant assumptions. We compare it to three existing paradigms: a baseline interface that emulates current image generation tools, a Gemini-style interface that produces “diverse” images, and an intention-centric interface that suggests aestheticized prompt refinements. We find that the agonistic interface enhances reflection across multiple measures, but also that reflection depends on authentically grounding diversity in relevant political contexts. Our results suggest that diversity and user intention need not be treated as inherently opposing values. Instead, interfaces can productively navigate tensions between competing perspectives, enabling users to engage with and evolve their intentions in meaningful ways.},
booktitle = {Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency},
pages = {438–463},
numpages = {26},
keywords = {agonism, reflection, diversity, factuality, image generation interfaces, human-AI interaction},
location = {
},
series = {FAccT '25}
}

@inproceedings{shi2025,
author = {Shi, Xinyu and Wang, Yinghou and Rossi, Ryan and Zhao, Jian},
title = {Brickify: Enabling Expressive Design Intent Specification through Direct Manipulation on Design Tokens},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3714087},
doi = {10.1145/3706598.3714087},
abstract = {Expressing design intent using natural language prompts requires designers to verbalize the ambiguous visual details concisely, which can be challenging or even impossible. To address this, we introduce Brickify, a visual-centric interaction paradigm — expressing design intent through direct manipulation on design tokens. Brickify extracts visual elements (e.g., subject, style, and color) from reference images and converts them into interactive and reusable design tokens that can be directly manipulated (e.g., resize, group, link, etc.) to form the visual lexicon. The lexicon reflects users’ intent for both what visual elements are desired and how to construct them into a whole. We developed Brickify to demonstrate how AI models can interpret and execute the visual lexicon through an end-to-end pipeline. In a user study, experienced designers found Brickify more efficient and intuitive than text-based prompts, allowing them to describe visual details, explore alternatives, and refine complex designs with greater ease and control.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {424},
numpages = {20},
keywords = {Design Intent Expression, Interaction Techniques, Direct Manipulation, Interactive Design Token},
location = {
},
series = {CHI '25}
}

@inproceedings{shin2024,
author = {Shin, Joonghyuk and Choi, Daehyeon and Park, Jaesik},
title = {InstantDrag: Improving Interactivity in Drag-based Image Editing},
year = {2024},
isbn = {9798400711312},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3680528.3687668},
doi = {10.1145/3680528.3687668},
abstract = {Drag-based image editing has recently gained popularity for its interactivity and precision. However, despite the ability of text-to-image models to generate samples within a second, drag editing still lags behind due to the challenge of accurately reflecting user interaction while maintaining image content. Some existing approaches rely on computationally intensive per-image optimization or intricate guidance-based methods, requiring additional inputs such as masks for movable regions and text prompts, thereby compromising the interactivity of the editing process. We introduce InstantDrag, an optimization-free pipeline that enhances interactivity and speed, requiring only an image and a drag instruction as input. InstantDrag consists of two carefully designed networks: a drag-conditioned optical flow generator (FlowGen) and an optical flow-conditioned diffusion model (FlowDiffusion). InstantDrag learns motion dynamics for drag-based image editing in real-world video datasets by decomposing the task into motion generation and motion-conditioned image generation. We demonstrate InstantDrag’s capability to perform fast, photo-realistic edits without masks or text prompts through experiments on facial video datasets and general scenes. These results highlight the efficiency of our approach in handling drag-based image editing, making it a promising solution for interactive, real-time applications.},
booktitle = {SIGGRAPH Asia 2024 Conference Papers},
articleno = {39},
numpages = {10},
keywords = {Interactive image editing, Drag-based image editing, GAN, Diffusion models, Optical flow},
location = {Tokyo, Japan},
series = {SA '24}
}

@inproceedings{son2024,
author = {Son, Kihoon and Choi, DaEun and Kim, Tae Soo and Kim, Young-Ho and Kim, Juho},
title = {GenQuery: Supporting Expressive Visual Search with Generative Models},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642847},
doi = {10.1145/3613904.3642847},
abstract = {Designers rely on visual search to explore and develop ideas in early design stages. However, designers can struggle to identify suitable text queries to initiate a search or to discover images for similarity-based search that can adequately express their intent. We propose  GenQuery, a novel system that integrates generative models into the visual search process. GenQuery can automatically elaborate on users’ queries and surface concrete search directions when users only have abstract ideas. To support precise expression of search intents, the system enables users to generatively modify images and use these in similarity-based search. In a comparative user study (N=16), designers felt that they could more accurately express their intents and find more satisfactory outcomes with  GenQuery compared to a tool without generative features. Furthermore, the unpredictability of generations allowed participants to uncover more diverse outcomes. By supporting both convergence and divergence, GenQuery led to a more creative experience.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {180},
numpages = {19},
keywords = {Creativity Support, Generative Model, Generative Search, Search Intent Expression, Visual Exploration, Visual Search},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{suckrow2024,
author = {Suckrow, Pierre-Louis Wolfgang L\'{e}on and Weber, Christoph Johannes and Rothe, Sylvia},
title = {Diffusion-Based Sound Synthesis in Music Production},
year = {2024},
isbn = {9798400710995},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3677996.3678289},
doi = {10.1145/3677996.3678289},
abstract = {In this paper, we explore the usability of generative artificial intelligence in music production through the development of a digital instrument that incorporates diffusion-based sound synthesis in its sound generation. Current text-to-audio models offer a novel method of defining sounds, which we aim to render utilizable in a music-production environment. Selected pretrained latent diffusion models, enable the synthesis of playable sounds through textual descriptions, which we incorporated into a digital instrument that integrates with standard music production tools. The resultant user interface not only allows generating but also modifying the sounds by editing model and instrument-specific parameters. We evaluated the applicability of current diffusion models with their parameters as well as the fitness of possible prompts for music production scenarios. Adapting published diffusion model pipelines for integration into the instrument, we facilitate experimentation and exploration of this innovative sound synthesis method. Our findings show that despite facing some limitations in the models' responsiveness to specific music production contexts and the instrument's functionality, the tool allows the development of novel and intriguing soundscapes. The instrument and code is published under https://github.com/suckrowPierre/WaveGenSynth},
booktitle = {Proceedings of the 12th ACM SIGPLAN International Workshop on Functional Art, Music, Modelling, and Design},
pages = {55–64},
numpages = {10},
keywords = {sound generation, text-to-sound, user interface, user study},
location = {Milan, Italy},
series = {FARM 2024}
}

@inproceedings{sung2025,
author = {Sung, Youjin and John, Kevin and Yoon, Sang Ho and Seifi, Hasti},
title = {HapticGen: Generative Text-to-Vibration Model for Streamlining Haptic Design},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3713609},
doi = {10.1145/3706598.3713609},
abstract = {Designing haptic effects is a complex, time-consuming process requiring specialized skills and tools. To support haptic design, we introduce HapticGen, a generative model designed to create vibrotactile signals from text inputs. We conducted a formative workshop to identify requirements for an AI-driven haptic model. Given the limited size of existing haptic datasets, we trained HapticGen on a large, labeled dataset of 335k audio samples using an automated audio-to-haptic conversion method. Expert haptic designers then used HapticGen’s integrated interface to prompt and rate signals, creating a haptic-specific preference dataset for fine-tuning. We evaluated the fine-tuned HapticGen with 32 users, qualitatively and quantitatively, in an A/B comparison against a baseline text-to-audio model with audio-to-haptic conversion. Results show significant improvements in five haptic experience (e.g., realism) and system usability factors (e.g., future use). Qualitative feedback indicates HapticGen streamlines the ideation process for designers and helps generate diverse, nuanced vibrations.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {495},
numpages = {24},
keywords = {Haptics, Designers, Generative AI, Extended Reality},
location = {
},
series = {CHI '25}
}

@inproceedings{tao2025,
author = {Tao, Sirui and Liang, Ivan and Peng, Cindy and Wang, Zhiqing and Palani, Srishti and Dow, Steven P.},
title = {DesignWeaver: Dimensional Scaffolding for Text-to-Image Product Design},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3714211},
doi = {10.1145/3706598.3714211},
abstract = {Generative AI has enabled novice designers to quickly create professional-looking visual representations for product concepts. However, novices have limited domain knowledge that could constrain their ability to write prompts that effectively explore a product design space. To understand how experts explore and communicate about design spaces, we conducted a formative study with 12 experienced product designers and found that experts — and their less-versed clients — often use visual references to guide co-design discussions rather than written descriptions. These insights inspired DesignWeaver, an interface that helps novices generate prompts for a text-to-image model by surfacing key product design dimensions from generated images into a palette for quick selection. In a study with 52 novices, DesignWeaver enabled participants to craft longer prompts with more domain-specific vocabularies, resulting in more diverse, innovative product designs. However, the nuanced prompts heightened participants’ expectations beyond what current text-to-image models could deliver. We discuss implications for AI-based product design support tools.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {425},
numpages = {26},
keywords = {Creativity support tools, design ideation, idea management, human-AI interaction, text-to-image models},
location = {
},
series = {CHI '25}
}

@inproceedings{tilekbay2024,
author = {Tilekbay, Bekzat and Yang, Saelyne and Lewkowicz, Michal Adam and Suryapranata, Alex and Kim, Juho},
title = {ExpressEdit: Video Editing with Natural Language and Sketching},
year = {2024},
isbn = {9798400705083},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3640543.3645164},
doi = {10.1145/3640543.3645164},
abstract = {Informational videos serve as a crucial source for explaining conceptual and procedural knowledge to novices and experts alike. When producing informational videos, editors edit videos by overlaying text/images or trimming footage to enhance the video quality and make it more engaging. However, video editing can be difficult and time-consuming, especially for novice video editors who often struggle with expressing and implementing their editing ideas. To address this challenge, we first explored how multimodality—natural language (NL) and sketching, which are natural modalities humans use for expression—can be utilized to support video editors in expressing video editing ideas. We gathered 176 multimodal expressions of editing commands from 10 video editors, which revealed the patterns of use of NL and sketching in describing edit intents. Based on the findings, we present ExpressEdit, a system that enables editing videos via NL text and sketching on the video frame. Powered by LLM and vision models, the system interprets (1) temporal, (2) spatial, and (3) operational references in an NL command and spatial references from sketching. The system implements the interpreted edits, which then the user can iterate on. An observational study (N=10) showed that ExpressEdit enhanced the ability of novice video editors to express and implement their edit ideas. The system allowed participants to perform edits more efficiently and generate more ideas by generating edits based on user’s multimodal edit commands and supporting iterations on the editing commands. This work offers insights into the design of future multimodal interfaces and AI-based pipelines for video editing.},
booktitle = {Proceedings of the 29th International Conference on Intelligent User Interfaces},
pages = {515–536},
numpages = {22},
keywords = {human-AI interaction, multimodal input, video editing},
location = {Greenville, SC, USA},
series = {IUI '24}
}

@inproceedings{vashishtha2024,
author = {Vashishtha, Shanu and Prakash, Abhinav and Morishetti, Lalitesh and Nag, Kaushiki and Arora, Yokila and Kumar, Sushant and Achan, Kannan},
title = {Chaining Text-to-Image and Large Language Model: A Novel Approach for Generating Personalized e-commerce Banners},
year = {2024},
isbn = {9798400704901},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3637528.3671636},
doi = {10.1145/3637528.3671636},
abstract = {Text-to-image models such as stable diffusion have opened a plethora of opportunities for generating art. Recent literature has surveyed the use of text-to-image models for enhancing the work of many creative artists. Many e-commerce platforms employ a manual process to generate the banners, which is time-consuming and has limitations of scalability. In this work, we demonstrate the use of text-to-image models for generating personalized web banners with dynamic content for online shoppers based on their interactions. The novelty in this approach lies in converting users' interaction data to meaningful prompts without human intervention. To this end, we utilize a large language model (LLM) to systematically extract a tuple of attributes from item meta-information. The attributes are then passed to a text-to-image model via prompt engineering to generate images for the banner. Our results show that the proposed approach can create high-quality personalized banners for users.},
booktitle = {Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining},
pages = {5825–5835},
numpages = {11},
keywords = {e-commerce, image generation, large language models, personalization, prompt engineering, stable diffusion},
location = {Barcelona, Spain},
series = {KDD '24}
}

@inproceedings{wang2023,
author = {Wang, Yunlong and Shen, Shuyuan and Lim, Brian Y},
title = {RePrompt: Automatic Prompt Editing to Refine AI-Generative Art Towards Precise Expressions},
year = {2023},
isbn = {9781450394215},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3544548.3581402},
doi = {10.1145/3544548.3581402},
abstract = {Generative AI models have shown impressive ability to produce images with text prompts, which could benefit creativity in visual art creation and self-expression. However, it is unclear how precisely the generated images express contexts and emotions from the input texts. We explored the emotional expressiveness of AI-generated images and developed RePrompt, an automatic method to refine text prompts toward precise expression of the generated images. Inspired by crowdsourced editing strategies, we curated intuitive text features, such as the number and concreteness of nouns, and trained a proxy model to analyze the feature effects on the AI-generated image. With model explanations of the proxy model, we curated a rubric to adjust text prompts to optimize image generation for precise emotion expression. We conducted simulation and user studies, which showed that RePrompt significantly improves the emotional expressiveness of AI-generated images, especially for negative emotions.},
booktitle = {Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems},
articleno = {22},
numpages = {29},
location = {Hamburg, Germany},
series = {CHI '23}
}

@inproceedings{wang2024a,
author = {Wang, Shaodong and Ge, Yunyang and Chen, Liuhan and Zhou, Haiyang and Wang, Qian and Cheng, Xinhua and Yuan, Li},
title = {Prompt2Poster: Automatically Artistic Chinese Poster Creation from Prompt Only},
year = {2024},
isbn = {9798400706868},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3664647.3681495},
doi = {10.1145/3664647.3681495},
abstract = {As a critical component in graphic design, artistic posters are widely applied in the advertising and entertainment industry, thus the automatic poster creation from user-provided prompts has become increasingly desired recently. Although existing Text2Image methods create impressive images aligned with given prompts, they fail to generate ideal artistic posters, especially with Chinese texts. To create desired artistic Chinese posters including an aligned background, reasonable layouts, and stylized graphical texts from given prompts only, we propose an automatic poster creation framework, named Prompt2Poster. Our framework utilizes the capacity of the powerful Large Language Model (LLM) to extract user intention from provided prompts and generate the aligned background. Although only taking a user prompt as the input, linguistic, visual, and geometrical information is fully utilized in the framework, bringing the ability to fit different distributions. To achieve the use of multi-modal information in the framework, two carefully designed modules, Controllable Layout Generator (CLG) and Graphical Text Generator (GTG) are proposed, leading to accurate and pleasurable visual results. Comprehensive experiments demonstrate that our Prompt2Poster achieves superior performance, especially in text quality and visual harmony.},
booktitle = {Proceedings of the 32nd ACM International Conference on Multimedia},
pages = {10716–10724},
numpages = {9},
keywords = {large language models, layout generation, stylized font generation, text to image generation},
location = {Melbourne VIC, Australia},
series = {MM '24}
}

@inproceedings{wang2024b,
author = {Wang, Zhijie and Huang, Yuheng and Song, Da and Ma, Lei and Zhang, Tianyi},
title = {PromptCharm: Text-to-Image Generation through Multi-modal Prompting and Refinement},
year = {2024},
isbn = {9798400703300},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3613904.3642803},
doi = {10.1145/3613904.3642803},
abstract = {The recent advancements in Generative AI have significantly advanced the field of text-to-image generation. The state-of-the-art text-to-image model, Stable Diffusion, is now capable of synthesizing high-quality images with a strong sense of aesthetics. Crafting text prompts that align with the model’s interpretation and the user’s intent thus becomes crucial. However, prompting remains challenging for novice users due to the complexity of the stable diffusion model and the non-trivial efforts required for iteratively editing and refining the text prompts. To address these challenges, we propose PromptCharm, a mixed-initiative system that facilitates text-to-image creation through multi-modal prompt engineering and refinement. To assist novice users in prompting, PromptCharm first automatically refines and optimizes the user’s initial prompt. Furthermore, PromptCharm supports the user in exploring and selecting different image styles within a large database. To assist users in effectively refining their prompts and images, PromptCharm renders model explanations by visualizing the model’s attention values. If the user notices any unsatisfactory areas in the generated images, they can further refine the images through model attention adjustment or image inpainting within the rich feedback loop of PromptCharm. To evaluate the effectiveness and usability of PromptCharm, we conducted a controlled user study with 12 participants and an exploratory user study with another 12 participants. These two studies show that participants using PromptCharm were able to create images with higher quality and better aligned with the user’s expectations compared with using two variants of PromptCharm that lacked interaction or visualization support.},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
articleno = {185},
numpages = {21},
keywords = {Generative AI, Large Language Models, Prompt Engineering},
location = {Honolulu, HI, USA},
series = {CHI '24}
}

@inproceedings{wang2025,
author = {Wang, Wen-Fan and Lee, Ting-Ying and Lu, Chien-Ting and Hsu, Che-Wei and Ponsa i Campany\`{a}, Nil and Chen, Yu and Chen, Mike Y. and Chen, Bing-Yu},
title = {GenTune: Toward Traceable Prompts to Improve Controllability of Image Refinement in Environment Design},
year = {2025},
isbn = {9798400720376},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3746059.3747774},
doi = {10.1145/3746059.3747774},
abstract = {Environment designers in the entertainment industry create imaginative 2D and 3D scenes for games, films, and television, requiring both fine-grained control of specific details and consistent global coherence. Designers have increasingly integrated generative AI into their workflows, often relying on large language models (LLMs) to expand user prompts for text-to-image generation, then iteratively refining those prompts and applying inpainting. However, our formative study with 10 designers surfaced two key challenges: (1) the lengthy LLM-generated prompts make it difficult to understand and isolate the keywords that must be revised for specific visual elements; and (2) while inpainting supports localized edits, it can struggle with global consistency and correctness. Based on these insights, we present GenTune, an approach that enhances human–AI collaboration by clarifying how AI-generated prompts map to image content. Our GenTune system lets designers select any element in a generated image, trace it back to the corresponding prompt labels, and revise those labels to guide precise yet globally consistent image refinement. In a summative study with 20 designers, GenTune significantly improved prompt-image comprehension, refinement quality and efficiency, and overall satisfaction (all p <.01) compared to current practice. A follow-up field study with two studios further demonstrated its effectiveness in real-world settings.},
booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology},
articleno = {11},
numpages = {21},
keywords = {Generative AI, Human-Centered AI, Environment Design, Creativity Support Tool, Visual Exploration, Traceable Prompt},
location = {
},
series = {UIST '25}
}

@inproceedings{wei2022,
author = {Wei, Yanheng and Huang, Lianghua and Zhang, Yanhao and Zheng, Yun and Pan, Pan},
title = {An Intelligent Advertisement Short Video Production System via Multi-Modal Retrieval},
year = {2022},
isbn = {9781450387323},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3477495.3536323},
doi = {10.1145/3477495.3536323},
abstract = {In its most basic form, advertising video production communicates a message about a product or service to the public. In the age of digital marketing, where the most popular way to connect with audiences is through advertising videos. However, advertising video production is a costly and complicated process from creation, material shooting, editing to the final commercial video. Therefore, producing qualified advertising videos is a capital and talent-intensive task, which poses a huge challenge for start-ups or inexperienced ad creators. paper proposes an intelligent advertising video production system driven by multi-modal retrieval, which only requires the input of descriptive copy. This system can automatically generate scripts, then extract key queries, retrieve related short video materials in the video library, and finally synthesize short advertising videos. The whole process minimizes human input, greatly reduces the threshold for advertising video production and greatly improves output and efficiency. It has a modular design to encourage the study of new multi-modal algorithms, which can be evaluated in batch mode. It can also integrate with a user interface, which allows user studies and data collection in an interactive mode, where the back end can be fully algorithmic or a wizard of oz setup. The proposed system has been fully verified and has broad prospects in the production of short videos for commodity advertisements within Alibaba.},
booktitle = {Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {3368–3372},
numpages = {5},
keywords = {video production, neural networks, multi-modal retrieval, cross-modal retrieval},
location = {Madrid, Spain},
series = {SIGIR '22}
}

@inproceedings{yan2025,
author = {Yan, Xun and Lu, Beier},
title = {Cultural Relic IP Generation Based On Structured Prompt And Hybrid Verification Methods},
year = {2025},
isbn = {9798400713927},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3749566.3749602},
doi = {10.1145/3749566.3749602},
abstract = {Aiming at the high threshold and cultural distortion faced by generative artificial intelligence technology in cultural heritage digitization, this paper proposes a cross-modal generation framework without programming environment, and realizes the high-fidelity generation and dissemination optimization of cultural heritage IP image through structured prompt and hybrid verification mechanism. This research method builds a three-level symbol control system based on cultural semiotics theory. MidJourney collaborates with large-scale language models to transform cultural relics features into structured prompt, and combines cross-modal narration to ensure historical logic consistency. Through social media A/B testing, the communication efficiency of the generated IP is significantly higher than that of traditional manual design, and the reference rate of cultural keywords is significantly improved, which promotes the innovative communication of cultural heritage among young groups, and provides a theoretical paradigm and practical reference in the interdisciplinary application field of AIGC.},
booktitle = {Proceedings of the 2025 5th International Conference on Internet of Things and Machine Learning},
pages = {161–165},
numpages = {5},
keywords = {AIGC, Hybrid verification, Image processing},
location = {
},
series = {IoTML '25}
}

@inproceedings{yoo2025,
author = {Yoo, Hojin and Nandi, Arnab},
title = {Emojis in Autocompletion: Enhancing Video Search with Visual Cues},
year = {2025},
isbn = {9798400719592},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3736733.3736745},
doi = {10.1145/3736733.3736745},
abstract = {Effective video search is increasingly challenging due to the inherent complexity and richness of video content, which traditional full-text query systems and text-based autocompletion methods struggle to capture. In this work, we propose an innovative autocompletion system that integrates visual cues, specifically, representative emojis, into the query formulation process to enhance video search efficiency. Our approach leverages cutting-edge Vision-Language Models (VLMs) to generate detailed scene descriptions from videos and employs Large Language Models (LLMs) to distill these descriptions into succinct, segmented search phrases augmented with context-specific emojis. A controlled user study, conducted with 11 university students using the MSVD dataset, demonstrates that the emoji-enhanced autocompletion reduces the average query completion time by 2.27 seconds (14.6% decrease) compared to traditional text-based methods, while qualitative feedback indicates mixed but generally positive user perceptions. These results highlight the potential of combining linguistic and visual modalities to redefine interactive video search experiences.},
booktitle = {Proceedings of the Workshop on Human-In-the-Loop Data Analytics},
articleno = {9},
numpages = {6},
keywords = {video search, autocompletion, emoji-enhanced interface},
location = {Intercontinental Berlin, Berlin, Germany},
series = {HILDA '25}
}

@inproceedings{zeng2024,
author = {Zeng, Chong and Dong, Yue and Peers, Pieter and Kong, Youkang and Wu, Hongzhi and Tong, Xin},
title = {DiLightNet: Fine-grained Lighting Control for Diffusion-based Image Generation},
year = {2024},
isbn = {9798400705250},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3641519.3657396},
doi = {10.1145/3641519.3657396},
abstract = {This paper presents a novel method for exerting fine-grained lighting control during text-driven diffusion-based image generation. While existing diffusion models already have the ability to generate images under any lighting condition, without additional guidance these models tend to correlate image content and lighting. Moreover, text prompts lack the necessary expressional power to describe detailed lighting setups. To provide the content creator with fine-grained control over the lighting during image generation, we augment the text-prompt with detailed lighting information in the form of radiance hints, i.e., visualizations of the scene geometry with a homogeneous canonical material under the target lighting. However, the scene geometry needed to produce the radiance hints is unknown. Our key observation is that we only need to guide the diffusion process, hence exact radiance hints are not necessary; we only need to point the diffusion model in the right direction. Based on this observation, we introduce a three stage method for controlling the lighting during image generation. In the first stage, we leverage a standard pretrained diffusion model to generate a provisional image under uncontrolled lighting. Next, in the second stage, we resynthesize and refine the foreground object in the generated image by passing the target lighting to a refined diffusion model, named DiLightNet, using radiance hints computed on a coarse shape of the foreground object inferred from the provisional image. To retain the texture details, we multiply the radiance hints with a neural encoding of the provisional synthesized image before passing it to DiLightNet. Finally, in the third stage, we resynthesize the background to be consistent with the lighting on the foreground object. We demonstrate and validate our lighting controlled diffusion model on a variety of text prompts and lighting conditions.},
booktitle = {ACM SIGGRAPH 2024 Conference Papers},
articleno = {73},
numpages = {12},
keywords = {Diffusion, Image Synthesis, Lighting Control, Radiance Hints},
location = {Denver, CO, USA},
series = {SIGGRAPH '24}
}

@article{zhang2023,
author = {Zhang, Yuxin and Dong, Weiming and Tang, Fan and Huang, Nisha and Huang, Haibin and Ma, Chongyang and Lee, Tong-Yee and Deussen, Oliver and Xu, Changsheng},
title = {ProSpect: Prompt Spectrum for Attribute-Aware Personalization of Diffusion Models},
year = {2023},
issue_date = {December 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3618342},
doi = {10.1145/3618342},
abstract = {Personalizing generative models offers a way to guide image generation with user-provided references. Current personalization methods can invert an object or concept into the textual conditioning space and compose new natural sentences for text-to-image diffusion models. However, representing and editing specific visual attributes such as material, style, and layout remains a challenge, leading to a lack of disentanglement and editability. To address this problem, we propose a novel approach that leverages the step-by-step generation process of diffusion models, which generate images from low to high frequency information, providing a new perspective on representing, generating, and editing images. We develop the Prompt Spectrum Space P*, an expanded textual conditioning space, and a new image representation method called ProSpect. ProSpect represents an image as a collection of inverted textual token embeddings encoded from per-stage prompts, where each prompt corresponds to a specific generation stage (i.e., a group of consecutive steps) of the diffusion model. Experimental results demonstrate that P* and ProSpect offer better disentanglement and controllability compared to existing methods. We apply ProSpect in various personalized attribute-aware image generation applications, such as image-guided or text-driven manipulations of materials, style, and layout, achieving previously unattainable results from a single image input without fine-tuning the diffusion models. Our source code is available at https://github.com/zyxElsa/ProSpect.},
journal = {ACM Trans. Graph.},
month = dec,
articleno = {244},
numpages = {14},
keywords = {attribute-aware editing, diffusion models, image generation, model personalization}
}

@article{zhang2024,
author = {Zhang, Zhiyuan and Chen, DongDong and Liao, Jing},
title = {SGEdit: Bridging LLM with Text2Image Generative Model for Scene Graph-based Image Editing},
year = {2024},
issue_date = {December 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {43},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3687957},
doi = {10.1145/3687957},
abstract = {Scene graphs offer a structured, hierarchical representation of images, with nodes and edges symbolizing objects and the relationships among them. It can serve as a natural interface for image editing, dramatically improving precision and flexibility. Leveraging this benefit, we introduce a new framework that integrates large language model (LLM) with Text2Image generative model for scene graph-based image editing. This integration enables precise modifications at the object level and creative recomposition of scenes without compromising overall image integrity. Our approach involves two primary stages: 1) Utilizing a LLM-driven scene parser, we construct an image's scene graph, capturing key objects and their interrelationships, as well as parsing fine-grained attributes such as object masks and descriptions. These annotations facilitate concept learning with a fine-tuned diffusion model, representing each object with an optimized token and detailed description prompt. 2) During the image editing phase, a LLM editing controller guides the edits towards specific areas. These edits are then implemented by an attention-modulated diffusion editor, utilizing the fine-tuned model to perform object additions, deletions, replacements, and adjustments. Through extensive experiments, we demonstrate that our framework significantly outperforms existing image editing methods in terms of editing precision and scene aesthetics. Our code is available at https://bestzzhang.github.io/SGEdit.},
journal = {ACM Trans. Graph.},
month = nov,
articleno = {195},
numpages = {16},
keywords = {image editing, scene graph, diffusion model}
}

@inproceedings{chen2024a,
author = {Chen, Helen Weixu and Istead, Lesley},
title = {"Imagine a Dress": Exploring the case of task-specific prompt assistants for text-to-image AI tools},
year = {2024},
isbn = {9798400718281},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3670947.3670972},
doi = {10.1145/3670947.3670972},
abstract = {In this paper, we explore the impact of task-specific prompt assistants for text-to-image generative AI tools through a user study. Participants were asked to recreate a dress with SDXL using either a prompt assistant tailored to the dress design, or, no assistant at all. A detailed analysis of the results and feedback suggests that for this specific task, a tailored assistant improves result satisfaction and accuracy. This style of assistant helps users focus on the task by providing a detailed, visual and organized approach to describing the object—enabling faster production times and more accurate descriptions with less ambiguity.},
booktitle = {Proceedings of the 50th Graphics Interface Conference},
articleno = {13},
numpages = {8},
keywords = {fashion, generative AI, image generation, prompt engineering},
location = {Halifax, NS, Canada},
series = {GI '24}
}