Experiments with Codex

Code Synthesis Evaluation of Codex using Python problem statements from HackerRank

01_intoduction

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#1	✅	➖	➖	➖
#2	✅	✅	❌	❌
#3	✅	✅	✅	➖
#4	✅	✅	❌	➖
#5	✅	➖	➖	➖
#6	✅	✅	➖	➖
#7	✅	✅	✅	➖

---

02_basic_data_types

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#8	✅	✅	✅	➖
#9	✅	✅	➖	❌
#10	✅	✅	➖	➖
#11	✅	✅	✅	➖
#12	✅	✅	✅	➖
#13	✅	➖	➖	➖

---

03_strings

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#14	✅	➖	➖	➖
#15	✅	➖	➖	❌
#16	✅	✅	✅	❌
#17	✅	✅	✅	❌
#18	✅	✅	✅	➖
#19	✅	✅	❌	➖
#20	✅	➖	➖	➖
#21	✅	✅	✅	➖
#22	✅	✅	❌	➖
#23	✅	✅	❌	➖
#24	✅	➖	➖	➖
#25	✅	➖	➖	➖
#26	✅	✅	✅	➖
#27	✅	✅	✅	➖

---

04_sets

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#28	✅	✅	❌	➖
#29	✅	✅	❌	➖
#30	✅	✅	➖	➖
#31	✅	✅	✅	➖
#32	✅	✅	✅	➖
#33	✅	✅	✅	➖
#34	✅	✅	✅	➖
#35	✅	✅	❌	➖
#36	✅	✅	❌	➖
#37	✅	✅	✅	❌
#38	✅	✅	✅	❌
#39	✅	❌	❌	➖
#40	✅	✅	❌	➖

---

05_math

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#41	✅	✅	➖	➖
#42	✅	✅	❌	➖
#43	✅	✅	❌	➖
#44	✅	✅	❌	❌
#45	✅	✅	✅	❌
#46	✅	✅	➖	❌
#47	✅	✅	❌	➖

---

06_itertools

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#48	✅	✅	❌	➖
#49	✅	✅	❌	➖
#50	✅	✅	❌	➖
#51	✅	✅	❌	➖
#52	✅	➖	❌	➖
#53	✅	❌	❌	➖
#54	✅	❌	❌	➖

---

07_collections

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#55	✅	✅	✅	➖
#56	✅	❌	❌	➖
#57	✅	✅	✅	➖
#58	✅	✅	❌	➖
#59	✅	✅	❌	➖
#60	✅	✅	➖	➖
#61	✅	✅	❌	➖
#62	✅	❌	❌	➖

---

08_date_and_time

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#63	✅	✅	➖	➖
#64	✅	✅	❌	➖

---

09_errors_and_exceptions

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#65	✅	❌	❌	➖
#66	✅	✅	❌	➖

---

10_classes

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#67	✅	✅	➖	➖
#68	✅	✅	❌	➖

---

11_builtins

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#69	✅	✅	✅	➖
#70	✅	❌	❌	➖
#71	✅	➖	➖	✅
#72	✅	✅	❌	➖
#73	✅	✅	✅	❌
#74	✅	➖	❌	➖

---

12_python_functionals

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#75	✅	➖	➖	✅
#76	✅	✅	❌	➖
#77	✅	✅	➖	➖

---

13_regex_and_parsing

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#78	✅	❌	❌	➖
#79	✅	➖	❌	➖
#80	✅	❌	❌	➖
#81	✅	✅	❌	➖
#82	✅	✅	❌	➖
#83	✅	✅	✅	❌
#84	✅	✅	➖	➖
#85	✅	✅	➖	➖
#86	✅	❌	❌	➖
#87	✅	❌	❌	➖
#88	✅	✅	✅	➖
#89	✅	✅	❌	➖
#90	✅	✅	❌	➖
#91	✅	✅	❌	❌
#92	✅	✅	✅	➖
#93	✅	✅	✅	➖
#94	✅	❌	❌	➖

---

14_xml

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#95	✅	✅	➖	➖
#96	✅	✅	➖	➖

---

15_closures_and_decorators

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#97	✅	❌	❌	➖
#98	✅	✅	✅	➖

---

16_numpy

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#99	✅	➖	❌	✅
#100	✅	✅	✅	❌
#101	✅	✅	✅	➖
#102	✅	✅	✅	➖
#103	✅	➖	➖	➖
#104	✅	✅	➖	➖
#105	✅	✅	✅	➖
#106	✅	✅	➖	➖
#107	✅	✅	✅	❌
#108	✅	✅	❌	❌
#109	✅	❌	❌	➖
#110	✅	✅	✅	❌
#111	✅	✅	❌	➖
#112	✅	❌	❌	➖
#113	✅	❌	❌	➖

---

17_debugging

problem_statement	full_problems	missing_specifications	missing_objectives	diff_objectives
#114	✅	✅	❌	➖
#115	✅	✅	❌	➖

---