MilanM commited on
Commit
e8dc4b0
·
verified ·
1 Parent(s): 075fa0c

Update helper_functions/image_helper_functions.py

Browse files
helper_functions/image_helper_functions.py CHANGED
@@ -1,11 +1,26 @@
1
  import pandas as pd
 
 
 
2
  import tempfile
3
  import mimetypes
4
  import base64
5
  import os
6
 
7
- def create_data_url(image_contents, filename):
8
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  if image_contents is None or filename is None:
10
  print("Usage: Provide image_contents and filename parameters")
11
  return
@@ -17,7 +32,366 @@ def create_data_url(image_contents, filename):
17
  mime_type = "application/octet-stream"
18
 
19
  return f"data:{mime_type};base64,{encoded_string}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
 
21
 
22
  if __name__ == "__main__":
23
  pass
 
1
  import pandas as pd
2
+ import marimo as mo
3
+ from PIL import Image
4
+ import pillow_heif
5
  import tempfile
6
  import mimetypes
7
  import base64
8
  import os
9
 
10
+ # --- --- --- --- --- --- ---
11
 
12
+ def create_data_url(image_contents, filename):
13
+ """
14
+ Convert image contents to a data URL format.
15
+
16
+ Args:
17
+ image_contents (bytes): The binary contents of the image file
18
+ filename (str): The filename used to determine the MIME type
19
+
20
+ Returns:
21
+ str: A data URL string in the format 'data:mime_type;base64,encoded_data'
22
+ """
23
+
24
  if image_contents is None or filename is None:
25
  print("Usage: Provide image_contents and filename parameters")
26
  return
 
32
  mime_type = "application/octet-stream"
33
 
34
  return f"data:{mime_type};base64,{encoded_string}"
35
+
36
+ # --- --- --- --- --- --- ---
37
+
38
+ def convert_heic_to_jpg(heic_path, output_path=None):
39
+ """
40
+ Convert HEIC file to JPEG format.
41
+
42
+ Args:
43
+ heic_path (str): Path to the input HEIC file
44
+ output_path (str, optional): Path for output JPEG file.
45
+ If None, uses same name with .jpg extension
46
+
47
+ Returns:
48
+ str: Path to the converted JPEG file
49
+ """
50
+
51
+ if output_path is None:
52
+ output_path = heic_path.rsplit(".", 1)[0] + ".jpg"
53
+
54
+ with Image.open(heic_path) as img:
55
+ rgb_image = img.convert("RGB")
56
+ rgb_image.save(output_path, "JPEG", quality=100)
57
+
58
+ return output_path
59
+
60
+ # --- --- --- --- --- --- ---
61
+
62
+ def create_image_example_message(instruction_prompt, file_example, output_text):
63
+ """
64
+ Create an example user message with image and corresponding assistant response.
65
+
66
+ Args:
67
+ instruction_prompt (str): The instruction prompt for processing images
68
+ file_example: File uploader object or tuple from .value
69
+ output_text (str): Expected output text for the example
70
+
71
+ Returns:
72
+ list: [user_message, assistant_message] pair for few-shot learning
73
+ """
74
+ if hasattr(file_example, "contents"):
75
+ file_contents = file_example.contents()
76
+ file_name = file_example.name()
77
+ else:
78
+ file_contents = file_example.contents(0)
79
+ file_name = file_example.name(0)
80
+
81
+ if file_name.lower().endswith(".heic"):
82
+ with tempfile.NamedTemporaryFile(
83
+ delete=False, suffix=".heic"
84
+ ) as temp_heic:
85
+ temp_heic.write(file_contents)
86
+ temp_heic_path = temp_heic.name
87
+
88
+ temp_jpg_path = convert_heic_to_jpg(temp_heic_path)
89
+
90
+ with open(temp_jpg_path, "rb") as jpg_file:
91
+ file_contents = jpg_file.read()
92
+
93
+ file_name = file_name.rsplit(".", 1)[0] + ".jpg"
94
+ os.unlink(temp_heic_path)
95
+ os.unlink(temp_jpg_path)
96
+
97
+ image_data_url = create_data_url(file_contents, file_name)
98
+
99
+ user_message = {
100
+ "role": "user",
101
+ "content": [
102
+ {"type": "text", "text": instruction_prompt},
103
+ {
104
+ "type": "image_url",
105
+ "image_url": {"url": image_data_url},
106
+ },
107
+ ],
108
+ }
109
+
110
+ assistant_message = {"role": "assistant", "content": output_text}
111
+
112
+ return [user_message, assistant_message]
113
+
114
+ # --- --- --- --- --- --- ---
115
+
116
+ def create_multiple_image_previews_with_conversion(
117
+ image_upload,
118
+ converted_files=None,
119
+ width=600,
120
+ use_stack_type="vstack",
121
+ ):
122
+ """
123
+ Create image previews for multiple uploaded files with proper HEIC conversion handling.
124
+ Args:
125
+ image_upload (mo.ui.file): The file uploader with multiple=True
126
+ converted_files (dict): Optional dict mapping index to converted file info
127
+ Format: {idx: {'contents': bytes, 'name': str}}
128
+ width (int): Optional Number of pixels for the maximum width of the image displays, defaults to 600
129
+ use_stack_type (str): If provided "vstack" it will return the images stacked vertically, otherwise it will be horizontal - "hstack"
130
+ Returns:
131
+ mo.vstack or mo.hstack: Stacked display of all image previews or "Add Images" message
132
+ """
133
+ if not image_upload.value:
134
+ return mo.md("Add Images")
135
+
136
+ image_previews = []
137
+
138
+ for idx in range(len(image_upload.value)):
139
+ original_name = image_upload.name(idx)
140
+ original_contents = image_upload.contents(idx)
141
+
142
+ if converted_files and idx in converted_files:
143
+ display_contents = converted_files[idx]["contents"]
144
+ display_name = converted_files[idx]["name"]
145
+ else:
146
+ if original_name.lower().endswith(".heic"):
147
+ with tempfile.NamedTemporaryFile(
148
+ delete=False, suffix=".heic"
149
+ ) as temp_heic:
150
+ temp_heic.write(original_contents)
151
+ temp_heic_path = temp_heic.name
152
+
153
+ temp_jpg_path = convert_heic_to_jpg(temp_heic_path)
154
+
155
+ with open(temp_jpg_path, "rb") as jpg_file:
156
+ display_contents = jpg_file.read()
157
+
158
+ display_name = original_name.rsplit(".", 1)[0] + ".jpg"
159
+
160
+ os.unlink(temp_heic_path)
161
+ os.unlink(temp_jpg_path)
162
+ else:
163
+ display_contents = original_contents
164
+ display_name = original_name
165
+
166
+ image_preview = mo.image(
167
+ src=display_contents,
168
+ rounded=True,
169
+ caption=display_name,
170
+ width=width,
171
+ )
172
+ image_previews.append(image_preview)
173
+
174
+ if use_stack_type == "vstack":
175
+ return mo.vstack(image_previews, align="center", gap=2)
176
+ else:
177
+ return mo.hstack(image_previews, justify="space-around")
178
+
179
+ # --- --- --- --- --- --- ---
180
+
181
+ def display_results_stack_with_data(
182
+ results_df, display_files, width=600, use_stack_type="vstack"
183
+ ):
184
+ """
185
+ Create display stack using pre-converted file data.
186
+
187
+ Args:
188
+ results_df (pd.DataFrame): Results with 'filename' and 'model_response'
189
+ display_files (dict): File data from process_multiple_images_with_display_data
190
+ width (int): Number of pixels for the maximum height of the image displays, defaults to 600
191
+ use_stack_type (str): If provided "vstack" it will return the images stacked vertically, otherwise it will be horizontal - "hstack"
192
+ Returns:
193
+ mo.vstack or mo.hstack: Stacked display of all image previews or "Add Images" message
194
+ """
195
+ if results_df is None or results_df.empty:
196
+ return mo.hstack(["", ""], justify="space-around", widths=[0.6, 0.4])
197
+
198
+ review_items = []
199
+
200
+ for idx, row in results_df.iterrows():
201
+ if idx in display_files:
202
+ file_data = display_files[idx]
203
+ image_preview = mo.image(
204
+ src=file_data["contents"],
205
+ rounded=True,
206
+ caption=file_data["name"],
207
+ width=width,
208
+ )
209
+ else:
210
+ image_preview = mo.md(f"**{row['filename']}**")
211
+
212
+ render_response = (
213
+ mo.md(f"{row['model_response']}")
214
+ if row["model_response"]
215
+ else mo.md("No response")
216
+ )
217
+
218
+ item_stack = mo.hstack(
219
+ [image_preview, render_response],
220
+ justify="space-around",
221
+ widths=[1.0, 0.3],
222
+ )
223
+ review_items.append(item_stack)
224
+
225
+ if use_stack_type == "vstack":
226
+ return mo.vstack(review_items, align="start", gap=2)
227
+ else:
228
+ return mo.hstack(review_items, justify="space-around")
229
+
230
+ # --- --- --- --- --- --- ---
231
+
232
+ def process_multiple_images_with_display_data(
233
+ instruction_prompt, image_uploader, chat_model
234
+ ):
235
+ """
236
+ Process multiple images and return both results and display-ready file data.
237
+
238
+ Args:
239
+ instruction_prompt (str): The instruction prompt for processing images
240
+ image_uploader (mo.ui.file): A marimo file uploader with multiple=True
241
+ chat_model: The initialized ModelInference chat model
242
+
243
+ Returns:
244
+ tuple: (pd.DataFrame, dict) - Results DataFrame and display file data
245
+ """
246
+
247
+ results = []
248
+ display_files = {}
249
+
250
+ for i in range(len(image_uploader.value)):
251
+ original_name = image_uploader.name(i)
252
+ file_contents = image_uploader.contents(i)
253
+ file_name = original_name
254
+
255
+ if original_name.lower().endswith(".heic"):
256
+ with tempfile.NamedTemporaryFile(
257
+ delete=False, suffix=".heic"
258
+ ) as temp_heic:
259
+ temp_heic.write(file_contents)
260
+ temp_heic_path = temp_heic.name
261
+
262
+ temp_jpg_path = convert_heic_to_jpg(temp_heic_path)
263
+
264
+ with open(temp_jpg_path, "rb") as jpg_file:
265
+ file_contents = jpg_file.read()
266
+
267
+ file_name = original_name.rsplit(".", 1)[0] + ".jpg"
268
+
269
+ display_files[i] = {
270
+ "contents": file_contents,
271
+ "name": file_name,
272
+ "original_name": original_name,
273
+ }
274
+
275
+ os.unlink(temp_heic_path)
276
+ os.unlink(temp_jpg_path)
277
+ else:
278
+ display_files[i] = {
279
+ "contents": file_contents,
280
+ "name": file_name,
281
+ "original_name": original_name,
282
+ }
283
+
284
+ image_data_url = create_data_url(file_contents, file_name)
285
+ messages = [
286
+ {
287
+ "role": "user",
288
+ "content": [
289
+ {"type": "text", "text": instruction_prompt},
290
+ {
291
+ "type": "image_url",
292
+ "image_url": {"url": image_data_url},
293
+ },
294
+ ],
295
+ }
296
+ ]
297
+
298
+ model_response = chat_model.chat(messages=messages)
299
+ model_image_response = model_response["choices"][0]["message"][
300
+ "content"
301
+ ]
302
+
303
+ results.append(
304
+ {"filename": file_name, "model_response": model_image_response}
305
+ )
306
+
307
+ return pd.DataFrame(results), display_files
308
+
309
+ # --- --- --- --- --- --- ---
310
+
311
+ def process_multiple_images_with_examples(
312
+ instruction_prompt, image_uploader, chat_model, example_message=None
313
+ ):
314
+ """
315
+ Process multiple images and return both results and display-ready file data.
316
+
317
+ Args:
318
+ instruction_prompt (str): The instruction prompt for processing images
319
+ image_uploader (mo.ui.file): A marimo file uploader with multiple=True
320
+ chat_model: The initialized ModelInference chat model
321
+ example (list): Example message+image input/output
322
+ Returns:
323
+ tuple: (pd.DataFrame, dict) - Results DataFrame and display file data
324
+ """
325
+
326
+ results = []
327
+ display_files = {}
328
+ example_messages = []
329
+
330
+ for i in range(len(image_uploader.value)):
331
+ original_name = image_uploader.name(i)
332
+ file_contents = image_uploader.contents(i)
333
+ file_name = original_name
334
+
335
+ if original_name.lower().endswith(".heic"):
336
+ with tempfile.NamedTemporaryFile(
337
+ delete=False, suffix=".heic"
338
+ ) as temp_heic:
339
+ temp_heic.write(file_contents)
340
+ temp_heic_path = temp_heic.name
341
+
342
+ temp_jpg_path = convert_heic_to_jpg(temp_heic_path)
343
+
344
+ with open(temp_jpg_path, "rb") as jpg_file:
345
+ file_contents = jpg_file.read()
346
+
347
+ file_name = original_name.rsplit(".", 1)[0] + ".jpg"
348
+
349
+ display_files[i] = {
350
+ "contents": file_contents,
351
+ "name": file_name,
352
+ "original_name": original_name,
353
+ }
354
+
355
+ os.unlink(temp_heic_path)
356
+ os.unlink(temp_jpg_path)
357
+ else:
358
+ display_files[i] = {
359
+ "contents": file_contents,
360
+ "name": file_name,
361
+ "original_name": original_name,
362
+ }
363
+
364
+ image_data_url = create_data_url(file_contents, file_name)
365
+
366
+ messages = []
367
+ if example_message:
368
+ messages.extend(example_messages)
369
+
370
+ messages.append(
371
+ {
372
+ "role": "user",
373
+ "content": [
374
+ {"type": "text", "text": instruction_prompt},
375
+ {
376
+ "type": "image_url",
377
+ "image_url": {"url": image_data_url},
378
+ },
379
+ ],
380
+ }
381
+ )
382
+
383
+ model_response = chat_model.chat(messages=messages)
384
+ model_image_response = model_response["choices"][0]["message"][
385
+ "content"
386
+ ]
387
+
388
+ results.append(
389
+ {"filename": file_name, "model_response": model_image_response}
390
+ )
391
+
392
+ return pd.DataFrame(results), display_files
393
 
394
+ # --- --- --- --- --- --- ---
395
 
396
  if __name__ == "__main__":
397
  pass