{ "name": "AI image-editing evaluation starter manifest", "version": "1.0", "published_at": "2026-08-11", "publisher": "Peelaway Engineering", "purpose": "Pipeline smoke test only. This 20-case manifest is too small to establish a winning system or a universal quality claim.", "source_notice": "This manifest links to third-party source images; it does not redistribute them. Resolve the relevant source record and comply with its attribution and license requirements before use.", "method": { "quality_axes": [ "instruction_following", "visual_fidelity_and_artifacts", "preservation" ], "comparison_protocol": "Blind pairwise review with A/B position swap; inconsistent preferences become ties and refusals remain separate.", "operational_metrics": [ "completion_rate", "failure_rate", "retry_rate", "end_to_end_latency_p50", "end_to_end_latency_p95", "cost_per_successful_output" ], "decision_boundary": "Pre-register the primary metric, minimum sample, uncertainty method, and critical no-regression slices before reviewing aggregate results. A powered decision needs substantially more than this starter set." }, "summary": { "cases": 20, "use_cases": 8, "edit_types": 7, "difficulty": { "easy": 3, "medium": 12, "hard": 5 } }, "cases": [ { "id": "re-01", "use_case": "real-estate", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/fa/Cozy_living_room_setting_with_a_yellow_sofa_and_a_coffee_table_adorned_with_cups_and_a_flower_pot.jpg/1920px-Cozy_living_room_setting_with_a_yellow_sofa_and_a_coffee_table_adorned_with_cups_and_a_flower_pot.jpg", "source_license": "CC BY 2.0", "instruction": "Recolor the mustard-yellow velvet sofa to a deep emerald-green velvet, matching the original fabric sheen and cushion folds.", "must_preserve": "coffee table, cups, flower vase, curtains, carpet, room layout and lighting", "difficulty": "easy" }, { "id": "re-02", "use_case": "real-estate", "edit_type": "material_swap", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/37/Stylish_living_room_features_a_comfortable_gray_sofa_and_wooden_paneling_under_a_high_ceiling.jpg/1920px-Stylish_living_room_features_a_comfortable_gray_sofa_and_wooden_paneling_under_a_high_ceiling.jpg", "source_license": "CC BY 2.0", "instruction": "Swap the wooden plank flooring for large light-gray marble tiles with subtle veining, following the room's perspective and reflections.", "must_preserve": "sofa, wood ceiling beams, mounted TV, fireplace, windows, furniture placement", "difficulty": "hard" }, { "id": "re-03", "use_case": "real-estate", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f5/Modern_kitchen_and_dining_area_with_stylish_furnishings_and_natural_light_in_a_contemporary_home_setting.jpg/1920px-Modern_kitchen_and_dining_area_with_stylish_furnishings_and_natural_light_in_a_contemporary_home_setting.jpg", "source_license": "CC BY 2.0", "instruction": "Recolor the charcoal-gray kitchen cabinets to a warm sage-green matte finish while leaving the countertop and appliances untouched.", "must_preserve": "countertops, oven and appliances, backsplash tiles, dining table and chairs, floor, windows", "difficulty": "easy" }, { "id": "ec-01", "use_case": "ecommerce", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/Witte_hoogsluitende_sportschoenen_met_donkerblauw%2C_witte_en_rode_details%2C_vetersluiting_met_daarover_een_ritssluiting_over_de_wreef_met_originele_doos%2C_objectnr_76173-2-3.JPG/1920px-thumbnail.jpg", "source_license": "CC BY-SA 3.0", "instruction": "Recolor the navy-blue side panels of both sneakers to bright teal, matching the existing lighting and leaving the white leather and red accents as they are.", "must_preserve": "shoe shape and proportions, white leather, red trim, laces and zipper hardware, gray background", "difficulty": "medium" }, { "id": "ec-02", "use_case": "ecommerce", "edit_type": "background_replace", "source_url": "https://upload.wikimedia.org/wikipedia/commons/3/39/Dog_rose_water.jpg", "source_license": "CC0", "instruction": "Replace the patterned blue-and-white tiled backdrop with a seamless soft-white studio background for a catalog listing.", "must_preserve": "both bottles, their liquid, caps and printed labels, their positions and scale", "difficulty": "medium" }, { "id": "po-01", "use_case": "photography-portrait", "edit_type": "replace_element", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/30/Russia%2C_Moscow_Oblast._Young_woman_with_umbrella%2C_studio_portrait.jpg/1920px-Russia%2C_Moscow_Oblast._Young_woman_with_umbrella%2C_studio_portrait.jpg", "source_license": "CC BY-SA 4.0", "instruction": "Replace the blue umbrella she is holding with a red Japanese paper parasol of a similar size and angle.", "must_preserve": "the woman's face, hair, blouse, waistcoat, seated pose, studio backdrop", "difficulty": "medium" }, { "id": "po-02", "use_case": "photography-portrait", "edit_type": "restyle", "source_url": "https://upload.wikimedia.org/wikipedia/commons/8/85/Portrait_of_Bearded_Man_by_Victor_Arimondi-NYC-1979.JPG", "source_license": "CC BY-SA 4.0", "instruction": "Restyle the portrait into a high-key black-and-white fine-art photograph with soft studio lighting.", "must_preserve": "the man's face, beard, tinted glasses, hand-on-chin pose", "difficulty": "medium" }, { "id": "fo-01", "use_case": "food", "edit_type": "add_element", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/eb/Hamburger_%28Burger_Bar%29.jpg/1920px-Hamburger_%28Burger_Bar%29.jpg", "source_license": "CC0", "instruction": "Add a melting slice of cheddar cheese draped over the patty and a couple of fresh green lettuce leaves under the top bun.", "must_preserve": "sesame bun, beef patty, sweet potato fries, plate, wooden table, skewer pick", "difficulty": "medium" }, { "id": "fo-02", "use_case": "food", "edit_type": "replace_element", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/48/Chicken_Quesadilla_dish_at_Latin_Bistro_restaurant_Summit_NJ.JPG/1920px-Chicken_Quesadilla_dish_at_Latin_Bistro_restaurant_Summit_NJ.JPG", "source_license": "CC0", "instruction": "Replace the white ceramic plate with a rustic round wooden serving board, keeping the quesadilla wedges and the three dip ramekins in place.", "must_preserve": "quesadilla wedges, the sour cream, guacamole and pico de gallo ramekins, food arrangement", "difficulty": "medium" }, { "id": "fo-03", "use_case": "food", "edit_type": "lighting", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/64/Restaurant_R%27Evolution%2C_New_Orleans_dish_2017.jpg/1920px-Restaurant_R%27Evolution%2C_New_Orleans_dish_2017.jpg", "source_license": "CC BY 2.0", "instruction": "Relight the dish as bright, crisp overhead food photography with soft daylight, brightening the plate and sauce.", "must_preserve": "the plated food, sauce and garnish, plate shape, glassware in the background", "difficulty": "hard" }, { "id": "ar-01", "use_case": "architecture", "edit_type": "lighting", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/9f/Old_Town_Architecture_Reflected_in_Modern_Facade_-_Geneva_-_Switzerland.jpg/1920px-Old_Town_Architecture_Reflected_in_Modern_Facade_-_Geneva_-_Switzerland.jpg", "source_license": "CC BY-SA 3.0", "instruction": "Change the time of day to golden-hour sunset, casting warm orange light and long reflections across the glass facade.", "must_preserve": "building structure, glass panels, the reflected clock tower, framing", "difficulty": "medium" }, { "id": "ar-02", "use_case": "architecture", "edit_type": "background_replace", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Robert_Mills_House_%28Ainsley_Hall_House%29%2C_Blanding_Street%2C_Columbia%2C_SC.jpg/1920px-Robert_Mills_House_%28Ainsley_Hall_House%29%2C_Blanding_Street%2C_Columbia%2C_SC.jpg", "source_license": "CC BY-SA 2.0", "instruction": "Replace the flat overcast sky with a vivid blue sky and golden sunset clouds behind the mansion.", "must_preserve": "the brick mansion, white portico columns, windows, staircase, lawn, hedges and trees", "difficulty": "medium" }, { "id": "tr-01", "use_case": "travel-landscape", "edit_type": "lighting", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/68/Himalayas%2C_Cholatse%2C_Nepal.jpg/1920px-Himalayas%2C_Cholatse%2C_Nepal.jpg", "source_license": "CC BY 4.0", "instruction": "Transform the scene to a pink-and-orange sunrise, tinting the snowy peaks with alpenglow and warming the valley light.", "must_preserve": "mountain silhouettes, valley, glacial lake, foreground rocks, composition", "difficulty": "medium" }, { "id": "tr-02", "use_case": "travel-landscape", "edit_type": "restyle", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Tropical_hue_of_the_Malindi_beach.jpg/1920px-Tropical_hue_of_the_Malindi_beach.jpg", "source_license": "CC BY-SA 4.0", "instruction": "Restyle the beach scene as a vibrant retro travel poster with bold flat colors and screen-print texture.", "must_preserve": "palm tree, shoreline, sea and horizon, rocks, overall layout", "difficulty": "medium" }, { "id": "tr-03", "use_case": "travel-landscape", "edit_type": "add_element", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/Mountains_in_snow%2C_Mountain_lake%2C_Chola_Valley%2C_Nepal%2C_Himalayas.jpg/1920px-Mountains_in_snow%2C_Mountain_lake%2C_Chola_Valley%2C_Nepal%2C_Himalayas.jpg", "source_license": "CC BY 4.0", "instruction": "Add gently falling snowflakes and a soft drifting mist across the valley.", "must_preserve": "the hikers, glacial pond, large boulder, peaks and snowfield", "difficulty": "medium" }, { "id": "au-01", "use_case": "automotive", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/28/Lamborghini_Revuelto_%2853877996613%29.jpg/1920px-Lamborghini_Revuelto_%2853877996613%29.jpg", "source_license": "CC BY 2.0", "instruction": "Recolor the car's silver-gray bodywork to a glossy candy-orange, keeping the black center stripe, red accents and wheels the same.", "must_preserve": "car body shape, badges, black center stripe, red trim, wheels, background crowd", "difficulty": "easy" }, { "id": "au-02", "use_case": "automotive", "edit_type": "lighting", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/bc/Vauxhall_Astra_K.jpg/1920px-Vauxhall_Astra_K.jpg", "source_license": "CC BY 4.0", "instruction": "Change the overcast daytime street to a night scene with the car's headlights and tail-lights glowing on a rain-slicked road.", "must_preserve": "car shape, color, wheels, badges, street and parked surroundings", "difficulty": "hard" }, { "id": "au-03", "use_case": "automotive", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/1953_Corvette_%286871665589%29.jpg/1920px-1953_Corvette_%286871665589%29.jpg", "source_license": "CC BY-SA 2.0", "instruction": "Recolor the classic convertible's white body to a vintage seafoam-teal, keeping the red interior and whitewall tires.", "must_preserve": "car shape, red interior, chrome trim, whitewall tires, show-floor background", "difficulty": "medium" }, { "id": "de-01", "use_case": "developers", "edit_type": "recolor", "source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d0/Bitwarden_Desktop_MacOS.png", "source_license": "GPLv3", "instruction": "Recolor the interface's blue accent elements - buttons, highlights and links - to a green palette while keeping all text and layout the same.", "must_preserve": "window chrome, sidebar labels, vault list entries, item fields, icons, text", "difficulty": "hard" }, { "id": "de-02", "use_case": "developers", "edit_type": "restyle", "source_url": "https://upload.wikimedia.org/wikipedia/commons/b/b3/Wikipedia_mobile_website_search_-_Brian_Johnson.png", "source_license": "CC BY-SA 4.0", "instruction": "Restyle the mobile web interface into a sleek dark-mode theme with a near-black background and light text.", "must_preserve": "all text and the search query, list entries, thumbnails, menu items, layout", "difficulty": "hard" } ] }