advaitgupta commited on
Commit
ee4cbbd
·
verified ·
1 Parent(s): 78f7447

Update aitw_gemini_dataset.json

Browse files
Files changed (1) hide show
  1. aitw_gemini_dataset.json +30 -30
aitw_gemini_dataset.json CHANGED
@@ -4,7 +4,7 @@
4
  "steps": [
5
  {
6
  "step_id": 0,
7
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_0.png",
8
  "action_history": "",
9
  "ground_truth_format": "mcq",
10
  "questions": {
@@ -20,7 +20,7 @@
20
  },
21
  {
22
  "step_id": 1,
23
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_1.png",
24
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]",
25
  "ground_truth_format": "mcq",
26
  "questions": {
@@ -36,7 +36,7 @@
36
  },
37
  {
38
  "step_id": 2,
39
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_2.png",
40
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]",
41
  "ground_truth_format": "mcq",
42
  "questions": {
@@ -52,7 +52,7 @@
52
  },
53
  {
54
  "step_id": 3,
55
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_3.png",
56
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]",
57
  "ground_truth_format": "mcq",
58
  "questions": {
@@ -68,7 +68,7 @@
68
  },
69
  {
70
  "step_id": 4,
71
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_4.png",
72
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]",
73
  "ground_truth_format": "mcq",
74
  "questions": {
@@ -84,7 +84,7 @@
84
  },
85
  {
86
  "step_id": 5,
87
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_5.png",
88
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]",
89
  "ground_truth_format": "mcq",
90
  "questions": {
@@ -100,7 +100,7 @@
100
  },
101
  {
102
  "step_id": 6,
103
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_6.png",
104
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]",
105
  "ground_truth_format": "mcq",
106
  "questions": {
@@ -116,7 +116,7 @@
116
  },
117
  {
118
  "step_id": 7,
119
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_7.png",
120
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]\nStep 6: Tapped at pixel coordinates (x,y): [514, 64]",
121
  "ground_truth_format": "mcq",
122
  "questions": {
@@ -132,7 +132,7 @@
132
  },
133
  {
134
  "step_id": 8,
135
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12172380859428428757_step_8.png",
136
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]\nStep 6: Tapped at pixel coordinates (x,y): [514, 64]\nStep 7: Tapped at pixel coordinates (x,y): [370, 127]",
137
  "ground_truth_format": "mcq",
138
  "questions": {
@@ -153,7 +153,7 @@
153
  "steps": [
154
  {
155
  "step_id": 0,
156
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_9027843537045096280_step_0.png",
157
  "action_history": "",
158
  "ground_truth_format": "mcq",
159
  "questions": {
@@ -169,7 +169,7 @@
169
  },
170
  {
171
  "step_id": 1,
172
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_9027843537045096280_step_1.png",
173
  "action_history": "Step 0: Pressed Button: Press Home",
174
  "ground_truth_format": "mcq",
175
  "questions": {
@@ -185,7 +185,7 @@
185
  },
186
  {
187
  "step_id": 2,
188
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_9027843537045096280_step_2.png",
189
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Tapped at pixel coordinates (x,y): [365, 826]",
190
  "ground_truth_format": "mcq",
191
  "questions": {
@@ -206,7 +206,7 @@
206
  "steps": [
207
  {
208
  "step_id": 0,
209
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_2247237893045084693_step_0.png",
210
  "action_history": "",
211
  "ground_truth_format": "mcq",
212
  "questions": {
@@ -222,7 +222,7 @@
222
  },
223
  {
224
  "step_id": 1,
225
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_2247237893045084693_step_1.png",
226
  "action_history": "Step 0: Swiped: Swipe Up",
227
  "ground_truth_format": "mcq",
228
  "questions": {
@@ -238,7 +238,7 @@
238
  },
239
  {
240
  "step_id": 2,
241
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_2247237893045084693_step_2.png",
242
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]",
243
  "ground_truth_format": "mcq",
244
  "questions": {
@@ -254,7 +254,7 @@
254
  },
255
  {
256
  "step_id": 3,
257
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_2247237893045084693_step_3.png",
258
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]\nStep 2: Tapped at pixel coordinates (x,y): [499, 976]",
259
  "ground_truth_format": "mcq",
260
  "questions": {
@@ -270,7 +270,7 @@
270
  },
271
  {
272
  "step_id": 4,
273
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_2247237893045084693_step_4.png",
274
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]\nStep 2: Tapped at pixel coordinates (x,y): [499, 976]\nStep 3: Tapped at pixel coordinates (x,y): [302, 950]",
275
  "ground_truth_format": "mcq",
276
  "questions": {
@@ -291,7 +291,7 @@
291
  "steps": [
292
  {
293
  "step_id": 0,
294
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_0.png",
295
  "action_history": "",
296
  "ground_truth_format": "mcq",
297
  "questions": {
@@ -307,7 +307,7 @@
307
  },
308
  {
309
  "step_id": 1,
310
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_1.png",
311
  "action_history": "Step 0: Swiped: Swipe Up",
312
  "ground_truth_format": "mcq",
313
  "questions": {
@@ -323,7 +323,7 @@
323
  },
324
  {
325
  "step_id": 2,
326
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_2.png",
327
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home",
328
  "ground_truth_format": "mcq",
329
  "questions": {
@@ -339,7 +339,7 @@
339
  },
340
  {
341
  "step_id": 3,
342
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_3.png",
343
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]",
344
  "ground_truth_format": "mcq",
345
  "questions": {
@@ -355,7 +355,7 @@
355
  },
356
  {
357
  "step_id": 4,
358
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_4.png",
359
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]\nStep 3: Typed: 'How much does a 2x4x8 board cost at Lowes?'",
360
  "ground_truth_format": "mcq",
361
  "questions": {
@@ -371,7 +371,7 @@
371
  },
372
  {
373
  "step_id": 5,
374
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_12224608131504749719_step_5.png",
375
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]\nStep 3: Typed: 'How much does a 2x4x8 board cost at Lowes?'\nStep 4: Tapped at pixel coordinates (x,y): [286, 129]",
376
  "ground_truth_format": "mcq",
377
  "questions": {
@@ -392,7 +392,7 @@
392
  "steps": [
393
  {
394
  "step_id": 0,
395
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_0.png",
396
  "action_history": "",
397
  "ground_truth_format": "mcq",
398
  "questions": {
@@ -408,7 +408,7 @@
408
  },
409
  {
410
  "step_id": 1,
411
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_1.png",
412
  "action_history": "Step 0: Pressed Button: Press Home",
413
  "ground_truth_format": "mcq",
414
  "questions": {
@@ -424,7 +424,7 @@
424
  },
425
  {
426
  "step_id": 2,
427
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_2.png",
428
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up",
429
  "ground_truth_format": "mcq",
430
  "questions": {
@@ -440,7 +440,7 @@
440
  },
441
  {
442
  "step_id": 3,
443
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_3.png",
444
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home",
445
  "ground_truth_format": "mcq",
446
  "questions": {
@@ -456,7 +456,7 @@
456
  },
457
  {
458
  "step_id": 4,
459
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_4.png",
460
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]",
461
  "ground_truth_format": "mcq",
462
  "questions": {
@@ -472,7 +472,7 @@
472
  },
473
  {
474
  "step_id": 5,
475
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_5.png",
476
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]\nStep 4: Typed: 'Google the capital of Mexico'",
477
  "ground_truth_format": "mcq",
478
  "questions": {
@@ -488,7 +488,7 @@
488
  },
489
  {
490
  "step_id": 6,
491
- "screenshot_path": "/n/fs/vision-mix/ag9604/aitw_gemini_images/episode_1307957808436696980_step_6.png",
492
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]\nStep 4: Typed: 'Google the capital of Mexico'\nStep 5: Tapped at pixel coordinates (x,y): [185, 114]",
493
  "ground_truth_format": "mcq",
494
  "questions": {
 
4
  "steps": [
5
  {
6
  "step_id": 0,
7
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_0.png",
8
  "action_history": "",
9
  "ground_truth_format": "mcq",
10
  "questions": {
 
20
  },
21
  {
22
  "step_id": 1,
23
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_1.png",
24
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]",
25
  "ground_truth_format": "mcq",
26
  "questions": {
 
36
  },
37
  {
38
  "step_id": 2,
39
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_2.png",
40
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]",
41
  "ground_truth_format": "mcq",
42
  "questions": {
 
52
  },
53
  {
54
  "step_id": 3,
55
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_3.png",
56
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]",
57
  "ground_truth_format": "mcq",
58
  "questions": {
 
68
  },
69
  {
70
  "step_id": 4,
71
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_4.png",
72
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]",
73
  "ground_truth_format": "mcq",
74
  "questions": {
 
84
  },
85
  {
86
  "step_id": 5,
87
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_5.png",
88
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]",
89
  "ground_truth_format": "mcq",
90
  "questions": {
 
100
  },
101
  {
102
  "step_id": 6,
103
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_6.png",
104
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]",
105
  "ground_truth_format": "mcq",
106
  "questions": {
 
116
  },
117
  {
118
  "step_id": 7,
119
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_7.png",
120
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]\nStep 6: Tapped at pixel coordinates (x,y): [514, 64]",
121
  "ground_truth_format": "mcq",
122
  "questions": {
 
132
  },
133
  {
134
  "step_id": 8,
135
+ "screenshot_path": "./aitw_gemini_images/episode_12172380859428428757_step_8.png",
136
  "action_history": "Step 0: Tapped at pixel coordinates (x,y): [382, 838]\nStep 1: Tapped at pixel coordinates (x,y): [224, 776]\nStep 2: Tapped at pixel coordinates (x,y): [420, 775]\nStep 3: Tapped at pixel coordinates (x,y): [468, 64]\nStep 4: Tapped at pixel coordinates (x,y): [44, 78]\nStep 5: Tapped at pixel coordinates (x,y): [467, 62]\nStep 6: Tapped at pixel coordinates (x,y): [514, 64]\nStep 7: Tapped at pixel coordinates (x,y): [370, 127]",
137
  "ground_truth_format": "mcq",
138
  "questions": {
 
153
  "steps": [
154
  {
155
  "step_id": 0,
156
+ "screenshot_path": "./aitw_gemini_images/episode_9027843537045096280_step_0.png",
157
  "action_history": "",
158
  "ground_truth_format": "mcq",
159
  "questions": {
 
169
  },
170
  {
171
  "step_id": 1,
172
+ "screenshot_path": "./aitw_gemini_images/episode_9027843537045096280_step_1.png",
173
  "action_history": "Step 0: Pressed Button: Press Home",
174
  "ground_truth_format": "mcq",
175
  "questions": {
 
185
  },
186
  {
187
  "step_id": 2,
188
+ "screenshot_path": "./aitw_gemini_images/episode_9027843537045096280_step_2.png",
189
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Tapped at pixel coordinates (x,y): [365, 826]",
190
  "ground_truth_format": "mcq",
191
  "questions": {
 
206
  "steps": [
207
  {
208
  "step_id": 0,
209
+ "screenshot_path": "./aitw_gemini_images/episode_2247237893045084693_step_0.png",
210
  "action_history": "",
211
  "ground_truth_format": "mcq",
212
  "questions": {
 
222
  },
223
  {
224
  "step_id": 1,
225
+ "screenshot_path": "./aitw_gemini_images/episode_2247237893045084693_step_1.png",
226
  "action_history": "Step 0: Swiped: Swipe Up",
227
  "ground_truth_format": "mcq",
228
  "questions": {
 
238
  },
239
  {
240
  "step_id": 2,
241
+ "screenshot_path": "./aitw_gemini_images/episode_2247237893045084693_step_2.png",
242
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]",
243
  "ground_truth_format": "mcq",
244
  "questions": {
 
254
  },
255
  {
256
  "step_id": 3,
257
+ "screenshot_path": "./aitw_gemini_images/episode_2247237893045084693_step_3.png",
258
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]\nStep 2: Tapped at pixel coordinates (x,y): [499, 976]",
259
  "ground_truth_format": "mcq",
260
  "questions": {
 
270
  },
271
  {
272
  "step_id": 4,
273
+ "screenshot_path": "./aitw_gemini_images/episode_2247237893045084693_step_4.png",
274
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Tapped at pixel coordinates (x,y): [379, 183]\nStep 2: Tapped at pixel coordinates (x,y): [499, 976]\nStep 3: Tapped at pixel coordinates (x,y): [302, 950]",
275
  "ground_truth_format": "mcq",
276
  "questions": {
 
291
  "steps": [
292
  {
293
  "step_id": 0,
294
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_0.png",
295
  "action_history": "",
296
  "ground_truth_format": "mcq",
297
  "questions": {
 
307
  },
308
  {
309
  "step_id": 1,
310
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_1.png",
311
  "action_history": "Step 0: Swiped: Swipe Up",
312
  "ground_truth_format": "mcq",
313
  "questions": {
 
323
  },
324
  {
325
  "step_id": 2,
326
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_2.png",
327
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home",
328
  "ground_truth_format": "mcq",
329
  "questions": {
 
339
  },
340
  {
341
  "step_id": 3,
342
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_3.png",
343
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]",
344
  "ground_truth_format": "mcq",
345
  "questions": {
 
355
  },
356
  {
357
  "step_id": 4,
358
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_4.png",
359
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]\nStep 3: Typed: 'How much does a 2x4x8 board cost at Lowes?'",
360
  "ground_truth_format": "mcq",
361
  "questions": {
 
371
  },
372
  {
373
  "step_id": 5,
374
+ "screenshot_path": "./aitw_gemini_images/episode_12224608131504749719_step_5.png",
375
  "action_history": "Step 0: Swiped: Swipe Up\nStep 1: Pressed Button: Press Home\nStep 2: Tapped at pixel coordinates (x,y): [289, 950]\nStep 3: Typed: 'How much does a 2x4x8 board cost at Lowes?'\nStep 4: Tapped at pixel coordinates (x,y): [286, 129]",
376
  "ground_truth_format": "mcq",
377
  "questions": {
 
392
  "steps": [
393
  {
394
  "step_id": 0,
395
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_0.png",
396
  "action_history": "",
397
  "ground_truth_format": "mcq",
398
  "questions": {
 
408
  },
409
  {
410
  "step_id": 1,
411
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_1.png",
412
  "action_history": "Step 0: Pressed Button: Press Home",
413
  "ground_truth_format": "mcq",
414
  "questions": {
 
424
  },
425
  {
426
  "step_id": 2,
427
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_2.png",
428
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up",
429
  "ground_truth_format": "mcq",
430
  "questions": {
 
440
  },
441
  {
442
  "step_id": 3,
443
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_3.png",
444
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home",
445
  "ground_truth_format": "mcq",
446
  "questions": {
 
456
  },
457
  {
458
  "step_id": 4,
459
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_4.png",
460
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]",
461
  "ground_truth_format": "mcq",
462
  "questions": {
 
472
  },
473
  {
474
  "step_id": 5,
475
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_5.png",
476
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]\nStep 4: Typed: 'Google the capital of Mexico'",
477
  "ground_truth_format": "mcq",
478
  "questions": {
 
488
  },
489
  {
490
  "step_id": 6,
491
+ "screenshot_path": "./aitw_gemini_images/episode_1307957808436696980_step_6.png",
492
  "action_history": "Step 0: Pressed Button: Press Home\nStep 1: Swiped: Swipe Up\nStep 2: Pressed Button: Press Home\nStep 3: Tapped at pixel coordinates (x,y): [320, 974]\nStep 4: Typed: 'Google the capital of Mexico'\nStep 5: Tapped at pixel coordinates (x,y): [185, 114]",
493
  "ground_truth_format": "mcq",
494
  "questions": {