BART_MLM_Continued_Pretraining.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","toc_visible":true,"collapsed_sections":["dDbYvKM8plEg"],"mount_file_id":"1UUhNF7m9IW6GPiw6tjVeAaKbIKKbva-P","authorship_tag":"ABX9TyOsIqkoY4RVTWDjObCJ0m/G"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"18a145b799a742718327d01f5a6095c6":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_158f9369c0fe41698af9487f5019131b","IPY_MODEL_dcbe36b519c04b69a119b9c5552d5f89","IPY_MODEL_7b292ed931a646b0bbedc0dfee07e4db"],"layout":"IPY_MODEL_8887f1e57e1a4492a68937185a15e4b6"}},"158f9369c0fe41698af9487f5019131b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_19ce3e5dccff43c28e4c1bc48c76f4a3","placeholder":"​","style":"IPY_MODEL_672227b42afc4af2b28e0cea9b49b0a5","value":"vocab.json: 100%"}},"dcbe36b519c04b69a119b9c5552d5f89":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d56003e2ecc541b8a31dee066ceaedb7","max":898823,"min":0,"orientation":"horizontal","style":"IPY_MODEL_833ecd615e2c43ce84473138482336e6","value":898823}},"7b292ed931a646b0bbedc0dfee07e4db":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ac893f761ace4e9ba2b84ffc628cbd42","placeholder":"​","style":"IPY_MODEL_838065164e7f4aac89e3aab0bba4129a","value":" 899k/899k [00:00&lt;00:00, 4.68MB/s]"}},"8887f1e57e1a4492a68937185a15e4b6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"19ce3e5dccff43c28e4c1bc48c76f4a3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"672227b42afc4af2b28e0cea9b49b0a5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d56003e2ecc541b8a31dee066ceaedb7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"833ecd615e2c43ce84473138482336e6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ac893f761ace4e9ba2b84ffc628cbd42":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"838065164e7f4aac89e3aab0bba4129a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ed9331de07c64c489e52bf1c6ad4f3aa":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_9195c31428a0400ea064f20510ad3c75","IPY_MODEL_1d5b158f1caf48a3a76542112119af5d","IPY_MODEL_81f9a9cc3ef9424ab3b8de09f406252b"],"layout":"IPY_MODEL_eacd577a91ab44db97070e6dc5582070"}},"9195c31428a0400ea064f20510ad3c75":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e9d74b7de3fb40ef9d04268594ed761c","placeholder":"​","style":"IPY_MODEL_e5b2962bfe7542f7bf8e321f0cc19896","value":"merges.txt: 100%"}},"1d5b158f1caf48a3a76542112119af5d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_2179f15774ec481d8689f73084fe22e9","max":456318,"min":0,"orientation":"horizontal","style":"IPY_MODEL_aa43a3a13f4942af9c4b4e2843c80597","value":456318}},"81f9a9cc3ef9424ab3b8de09f406252b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_021a611cda154097a7a1abdcbea354f2","placeholder":"​","style":"IPY_MODEL_1f15fd15fecf4c55896e44a8f2118b25","value":" 456k/456k [00:00&lt;00:00, 5.91MB/s]"}},"eacd577a91ab44db97070e6dc5582070":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e9d74b7de3fb40ef9d04268594ed761c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e5b2962bfe7542f7bf8e321f0cc19896":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2179f15774ec481d8689f73084fe22e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aa43a3a13f4942af9c4b4e2843c80597":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"021a611cda154097a7a1abdcbea354f2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1f15fd15fecf4c55896e44a8f2118b25":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ce64ceecad8346bea68870f62ad5bc43":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_9c89b3b9b4754132a3beb6c9585ea415","IPY_MODEL_f3ceaa229f854b4aa12c8a3de67bbc64","IPY_MODEL_68ae4448f561485e815aa0742709bffc"],"layout":"IPY_MODEL_841e1bd4a48a4c0db54598f36b560794"}},"9c89b3b9b4754132a3beb6c9585ea415":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_95b8fba9d5c3470baba50d8452c1181a","placeholder":"​","style":"IPY_MODEL_f207de4171724d9098c40f4290df06b9","value":"tokenizer.json: 100%"}},"f3ceaa229f854b4aa12c8a3de67bbc64":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8e5f27409e144249af7d32b73f520e75","max":1355863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2d3a7e80f9294b11a985afd39bb3187a","value":1355863}},"68ae4448f561485e815aa0742709bffc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a6e965839d3b4240830612c340add746","placeholder":"​","style":"IPY_MODEL_1dcd5e0210af4b90863586178ae80f1b","value":" 1.36M/1.36M [00:00&lt;00:00, 4.16MB/s]"}},"841e1bd4a48a4c0db54598f36b560794":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"95b8fba9d5c3470baba50d8452c1181a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f207de4171724d9098c40f4290df06b9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8e5f27409e144249af7d32b73f520e75":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2d3a7e80f9294b11a985afd39bb3187a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a6e965839d3b4240830612c340add746":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1dcd5e0210af4b90863586178ae80f1b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"960e28e1db0d4eba85c020370e5c260b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_4f97525ece9a4743b0c9a6834d49ce1a","IPY_MODEL_c86a1587136248348188069d51b6d212","IPY_MODEL_8283c30986e44cf4a921b73bd43bc5ec"],"layout":"IPY_MODEL_ad31db176dc64b518a2599404ce0573b"}},"4f97525ece9a4743b0c9a6834d49ce1a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8a0e2c61ce58473c9de54270a9717e73","placeholder":"​","style":"IPY_MODEL_cb0468aac45849cfaf4378c1db26114d","value":"config.json: 100%"}},"c86a1587136248348188069d51b6d212":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c5f5ba16b4b74221bb1924afc60b7517","max":1716,"min":0,"orientation":"horizontal","style":"IPY_MODEL_ebda7f4d336541f7a651ebdfa7855c04","value":1716}},"8283c30986e44cf4a921b73bd43bc5ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_487864fbf08049b8ad4c725be03537a9","placeholder":"​","style":"IPY_MODEL_ddb998fa9cab4671b55c9473579e5ac3","value":" 1.72k/1.72k [00:00&lt;00:00, 55.8kB/s]"}},"ad31db176dc64b518a2599404ce0573b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8a0e2c61ce58473c9de54270a9717e73":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cb0468aac45849cfaf4378c1db26114d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c5f5ba16b4b74221bb1924afc60b7517":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ebda7f4d336541f7a651ebdfa7855c04":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"487864fbf08049b8ad4c725be03537a9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ddb998fa9cab4671b55c9473579e5ac3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6f24c4de0bf24248b6d333a2b14447ce":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_45b78cdf80464175993f818c67d0fd98","IPY_MODEL_19f715289d6b4c66b4432e398e3be9ed","IPY_MODEL_fa137ae3569d499f8b47112e25a88833"],"layout":"IPY_MODEL_e47cf5e886f7442586e50da66081ffa5"}},"45b78cdf80464175993f818c67d0fd98":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7d040032a4db4c42836e27cc9bef20e7","placeholder":"​","style":"IPY_MODEL_b12fa8b54d9148a599373f2650b2e047","value":"model.safetensors: 100%"}},"19f715289d6b4c66b4432e398e3be9ed":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_dd9ea816f7ad405780b6e65d3896cdc3","max":557709915,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8c791a6b35944dea9318188782731a51","value":557709915}},"fa137ae3569d499f8b47112e25a88833":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0b1e85f75991462f81b71f7fed0b0d81","placeholder":"​","style":"IPY_MODEL_a10f5bd2c13f452dbf7c0c07d5d6d9e1","value":" 558M/558M [00:03&lt;00:00, 168MB/s]"}},"e47cf5e886f7442586e50da66081ffa5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7d040032a4db4c42836e27cc9bef20e7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b12fa8b54d9148a599373f2650b2e047":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"dd9ea816f7ad405780b6e65d3896cdc3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8c791a6b35944dea9318188782731a51":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0b1e85f75991462f81b71f7fed0b0d81":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a10f5bd2c13f452dbf7c0c07d5d6d9e1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}},"accelerator":"GPU"},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"0nYNofK3Zz-r","executionInfo":{"status":"ok","timestamp":1712422174425,"user_tz":240,"elapsed":14050,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"outputs":[],"source":["import os\n","import time\n","import copy\n","import random\n","\n","import torch\n","import numpy as np\n","import pandas as pd\n","\n","from sklearn.model_selection import train_test_split\n","\n","from torch.utils.data import Dataset, DataLoader\n","from torch import nn\n","from torch.nn.functional import one_hot\n","\n","from transformers import AutoTokenizer, AutoModel, AutoModelForSeq2SeqLM\n","\n","from IPython.display import HTML, display"]},{"cell_type":"markdown","source":["## Workspace Setup"],"metadata":{"id":"xRzKHNWMYBr0"}},{"cell_type":"code","source":["# ----------------------- #\n","# --- Workspace Setup --- #\n","# ----------------------- #\n","\n","# set project directory\n","# -----------------------\n","project_folder = 'drive/MyDrive/Datasci266/w266_project/'\n","os.chdir(project_folder)\n","\n","# set device\n","# -----------------------\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","# wrap cell outputs\n","# -----------------------\n","def set_css():\n","  display(HTML('''\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  '''))\n","get_ipython().events.register('pre_run_cell', set_css)"],"metadata":{"id":"eIMPwCjcX-nY","executionInfo":{"status":"ok","timestamp":1712422174425,"user_tz":240,"elapsed":18,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":["## Load Raw Training Data"],"metadata":{"id":"cp1M08u_YIqB"}},{"cell_type":"code","source":["# -------------------------- #\n","# --- Load Training Data --- #\n","# -------------------------- #\n","\n","df = pd.read_pickle('data/train_filtered.pkl')\n","df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":293},"id":"lfPRF1LxVyi0","executionInfo":{"status":"ok","timestamp":1712422180484,"user_tz":240,"elapsed":6076,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"d40f68e5-2a76-4721-e2ef-5b50c7cf2f23"},"execution_count":3,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["                                             article  \\\n","0  with significant research efforts being direct...   \n","1  the open connectome project ( located at http:...   \n","2  i am grateful to alekos kechris for informing ...   \n","3  set theory was proposed with the intended use ...   \n","4  this work is financially supported by the nati...   \n","\n","                                            abstract  \\\n","0   synaptic memory is considered to be the main ...   \n","1   * _ abstract _ : * in this paper , we present...   \n","2   we describe the fundamental constructions and...   \n","3   recently , a multi - level fuzzy min max neur...   \n","4   we review the direct cp and t violation in th...   \n","\n","                                       section_names  \n","0  introduction\\nformalism\\nresults and discussio...  \n","1   introduction\\nprocedure\\nresults and future work  \n","2                                    acknowledgments  \n","3  introduction\\nmulti-level fuzzy min-max neural...  \n","4                                   acknowledgements  "],"text/html":["\n","  <div id=\"df-1665b88c-4332-45b7-8359-04d7b0b14d30\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>article</th>\n","      <th>abstract</th>\n","      <th>section_names</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>with significant research efforts being direct...</td>\n","      <td>synaptic memory is considered to be the main ...</td>\n","      <td>introduction\\nformalism\\nresults and discussio...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>the open connectome project ( located at http:...</td>\n","      <td>* _ abstract _ : * in this paper , we present...</td>\n","      <td>introduction\\nprocedure\\nresults and future work</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>i am grateful to alekos kechris for informing ...</td>\n","      <td>we describe the fundamental constructions and...</td>\n","      <td>acknowledgments</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>set theory was proposed with the intended use ...</td>\n","      <td>recently , a multi - level fuzzy min max neur...</td>\n","      <td>introduction\\nmulti-level fuzzy min-max neural...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>this work is financially supported by the nati...</td>\n","      <td>we review the direct cp and t violation in th...</td>\n","      <td>acknowledgements</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1665b88c-4332-45b7-8359-04d7b0b14d30')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-1665b88c-4332-45b7-8359-04d7b0b14d30 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-1665b88c-4332-45b7-8359-04d7b0b14d30');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-cbcce372-c216-4889-9982-6f24953422d8\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cbcce372-c216-4889-9982-6f24953422d8')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-cbcce372-c216-4889-9982-6f24953422d8 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 2418,\n  \"fields\": [\n    {\n      \"column\": \"article\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2403,\n        \"samples\": [\n          \"gamma ray astronomy at energies around 100 @xmath0 - 10 @xmath1 is the main scientific goal of the argo - ybj experiment @xcite .\\nthe detector , which is now being assembled in tibet ( china ) at 4300 @xmath2 a.s.l .\\n, is a full coverage extensive air shower array consisting of a resistive plate chamber ( rpc ) carpet of more than 6000 @xmath3 .\\nit is logically divided into 154 units called _ clusters _ ( @xmath4 m@xmath5 ) , made by 12 rpcs ( see fig.[fig : argo ] ) .\\neach rpc ( @xmath6 m@xmath5 ) is read out by 10 pads ( @xmath7 cm@xmath5 ) , which are further divided into 8 different strips ( @xmath9 cm@xmath5 ) , which provide the highest available space resolution . the signals coming from all the strips of a given pad\\nare sent to the same channel of a multihit tdc .\\nthe whole system is designed in order to provide a single hit time resolution at the level of 1 ns , thus allowing a complete and detailed three - dimensional reconstruction of the shower front . the high altitude\\n( @xmath10 g / cm@xmath5 ) and the full coverage ensure a very low primary energy threshold ( e@xmath11100 gev ) , while the detector time resolution gives a good pointing accuracy , thus allowing a high sensitivity to @xmath12-ray sources .\\ngamma / hadron discrimination is a key issue in very high energy ( vhe ) gamma ray astronomy since it allows , together with a good angular resolution , the rejection of the huge background due to charged primary hadrons .\\nthe use of a full coverage detector with a high space granularity - like argo - ybj - can give detailed images of the shower front .\\nintrinsic differences in the processes involved in the electromagnetic and hadronic shower development in the atmosphere can then be evidenced by means of a careful analysis of the event@xcite .\\nrecently the use of multiscale behavior of event images has been showed to give good results in experiments exploiting the imaging atmospheric cherenkov techniques ( iact ) , together with the use of the so - called hillas parameters @xcite . in these experiments\\nthe image is integrated over the entire shower development , while in the case of argo - ybj a section of the shower is provided at a given ( slanted ) depth only , thus giving a potential lack of information .\\nhowever the typical disuniformities present in hadronic events might be better evidenced in the case of argo - ybj , these being partially masked in the case of iact detectors because of the smearing effect due to the integration of the information along the shower development .    in this work event images\\nhave been analyzed at different length scales and their multifractal nature has been studied . in particular\\nthe discrete wavelet transforms have been applied since they allowed a differential approach to multifractality , that gave a higher discrimination power .    -0.5\\nwe have considered the shower image seen by argo - ybj as a function @xmath13 defined on a two - dimensional space and corresponding to the amplitudes given by the measured strip multiplicity . as a first step we can calculate the multifractal moment @xmath14 of order q , defined ( see @xcite ) at the length scale @xmath15 as : @xmath16 where @xmath17 gives the probability for a hit to be in the box @xmath18 of dimension @xmath15 centered at @xmath19 , being @xmath20 the total content of the image . by dividing the image , at different steps , into non overlapping pixels of size @xmath15\\n, we can evaluate the box - amplitudes @xmath21 . as pointed out in @xcite , at each order @xmath22 , the mf moment is expected to have a power law dependence on @xmath15 in the high resolution limit , namely : @xmath23 for @xmath24 .    by fitting the behavior of @xmath14 on @xmath15 ( for each value of @xmath22 )\\nthe mf scaling exponent @xmath25 can be extracted .\\nthe dependence of @xmath25 on @xmath22 gives the main information on the mf properties of the image .\\nas shown in @xcite , the mf approach might not sufficiently characterize the image .\\nan approach to multifractality based on the discrete wavelet transformations ( dwt ) is more appropriate .\\ndwt can be seen as an expansion of @xmath13 on a discrete set of basis functions that are generated by scaling a so - called mother wavelet .\\nfor sake of simplicity , let us consider a one - dimensional case .\\nif the haar mother wavelet @xmath26 is chosen ( see fig.[fig : haar ] ) , the differences between the box amplitudes of two adjacent cells can be given by the proper convolution : @xmath27 in this simple case , the mother wavelet can be indicated as ( + @xmath28 - ) . a differential approach to multifractality\\nis then given by defining the dwt moment as : @xmath29 also in this case the scaling properties of the image can be evidenced at high resolutions : @xmath30 when @xmath24 .\\n-0.7 cm    in the case of a two - dimensional distribution ( i.e. an image ) we need three base wavelets that can be indicated as : @xmath31 this led to write three different dwt moments @xmath32 , @xmath33 , defined as : @xmath34 these functions are expected to scale like @xmath35 , when @xmath24 , with different exponents @xmath36 , whose dependence on @xmath22 gives the maximum information on the image properties .\\nwe have generated @xmath372.8@xmath3810@xmath39 gamma - initiated showers and @xmath372.6@xmath3810@xmath39 proton - initiated ones making use of the corsika code @xcite .\\nthe events have been taken within the energy range 30 gev@xmath40100 tev with azimuth between 0 and 15 degrees and core at the detector center .\\nthe primary energy spectrum has been generated according to the measured power laws with spectral index @xmath12=2.5 for gammas and @xmath12=2.7 for hadrons .\\nthe detector response has been fully simulated by using argog , a tool ( based on the geant3 package @xcite ) developed within the argo - ybj collaboration .\\nsince it is the hit multiplicity which is actually measured in eas experiments , it is correct and appropriate to classify events following this variable and not in terms of the primary energy . therefore we divided the simulated data sample into five multiplicity windows as reported in tab.[tab : ann ] , where the average primary energies are also shown .\\nthe @xmath12 and proton energies are obviously different since they depend on the processes involved in the shower development in the atmosphere . in particular proton - induced showers results in lower multiplicity events with respect to the case of photon - induced ones with the same primary energy .\\nmoreover in order to avoid distortion effects due to the finite size of the energy window in which the samples have been simulated , we have considered in the analysis only events which produced more than 50 and less than 6000 hits .\\nthis corresponds to average primary photon energies between @xmath41gev and @xmath42tev ( see tab.[tab : ann ] ) .\\nthe multiresolution quantities defined in sec.[sec : multiscale ] have been used to analyze each event image . as reported in sec.[sec : intro ] the argo - ybj detector is made by a central carpet and a guard ring ( see fig.[fig : argo ] ) . in order to preserve the same symmetry at different\\nlenght scales @xmath15 , we decided to neglect , in this first analysis , the information coming from the external ring .\\nwe also decided to _ mask _ the central carpet with a square grid .\\nin particular , since it is made of ( 120@xmath43130)pads , the first and the last row of pads were not considered , while four empty columns of pads were added ( two on the left and two on the right ) , thus obtaining a ( 128@xmath43128 ) pad mask . in order to limit statistical fluctuations of the hit multiplicity in the smallest pixels ,\\nthe minimum pixel size considered in the analysis was set at ( 2@xmath432)pads - about @xmath44m@xmath5 - which will then corresponds to the maximum resolution , i.e. @xmath15=1 . the analysis of an event goes then throught different steps , each correponding to different lenght scales . at the @xmath45-th step ( with @xmath46 ) , the image is divided into 2@xmath472@xmath48 square pixels of size @xmath49 , containing each @xmath50 pads , and the total strip multiplicity is computed in each considered pixel .\\ncc    &    4.cm    the values of log(z@xmath51(@xmath15 ) ) and log(w@xmath51(@xmath15 ) ) have been calculated following eq.[eq : zq ] and eq.[eq : wq ] , and their dependences on log(@xmath15 ) have been fitted with a first order polynomial in the region where the scaling is expected ( i.e. @xmath24 ) , for different values of the moment @xmath22 .\\nthe scaling exponents @xmath52(q ) and @xmath53(q ) , defined in sec.[sec : multiscale ] , have then been obtained for each event .\\ntheir dependences on the moment @xmath22 are shown in fig.[fig : scalingvsq ] for a sample of gamma and proton initiated events .\\nas can be seen there is a separation of the average values of the scaling exponents between electromagnetic and hadronic showers , which is however partly masked by the large fluctuations .\\ntherefore the separations of the mf and dwt parameters are not sufficient in order to give a good discrimination between e.m . and hadronic showers , unless an artificial neural network ( ann ) is used as in ref.@xcite .\\nwe then decided to use multifractal parameters as inputs to a properly designed and trained ann ( see sec.[sec : ann ] ) .    in order to increase the @xmath12/h separation , a study on the shape and the simmetry of the event image\\nhas also been made .\\nin particular we studied the skewness of each event by means of the third moment of the distributions of the hit coordinates on the detector plane , namely @xmath54 and @xmath55 .\\nin general , the skewness of a sequence of values @xmath56 is defined as : @xmath57 where @xmath58 and @xmath59 are the average and r.m.s . over @xmath60 , respectively . in our case\\nwe studied the behaviour of the quantity @xmath61 , with : @xmath62 where @xmath63 is the strip multiplicity of the pad at the position @xmath64 .\\nthe value of @xmath65 is reported in fig.[fig : xi ] for two samples of e.m . and hadronic showers in two different multiplicity windows .\\nas can be seen , its average is almost one as expected from both its definition and the detector geometry , while the r.m.s is always larger for proton - initiated showers , due to the large fluctuations present in hadronic events .\\nthis behaviour suggested the use of @xmath65 as one of the inputs to the ann .\\ncc    &    4.cm\\nin order to perform the @xmath12/hadron discrimination using the parameters we have introduced in the previous sections , we decided to make use of an artificial neural network .\\nthe neural network we have chosen is of the _ feed forward _ type and it is made of 3 perceptrons layers .\\nthe ann input is an eight - dimensional vector whose elements are : the event total hit multiplicity @xmath66 , the value of @xmath65 , the multifractal exponents @xmath52 and @xmath53 for @xmath67 . the output vector is defined in a one dimensional space : it is trained to be 1 for gamma - initiated events and 0 for hadronic showers .    cc    &    5.cm    networks were implemented and optimized by using the * s*tuttgart * n*eural * n*etwork * s*imulator ( snns ) tool @xcite . in designing the ann , its characteristics were been deeply studied in order to reach a good compromise between the increase of recognition capability and the processing time . the network training was separately performed in 5 multiplicity windows by using several thousands events ( see tab.[tab : ann ] ) .\\nthe ann were then tested by using an independent reduced sample of events and the @xmath12 recognition efficiency @xmath68 , together with the proton contamination @xmath69 , were measured .\\nthe quantities @xmath68 and @xmath70 reached their plateau value after few thousands of ann training epochs .\\nan example of ann output for a couple of multiplicity windows is shown in fig.[fig : ann ] .    the detector sensitivity to @xmath12-ray sources\\nis defined as @xmath71/@xmath72 , where @xmath73 is the number of gamma - initiated events , while @xmath74 is the hadron contamination of the considered sample . the use of a @xmath12/h discrimination tool like the one we are considering in this work makes the sensitivity @xmath75 to be multiplied by the factor @xmath76/@xmath77 .    in this\\nwork values of @xmath782 have been reached ( see tab.[tab : ann ] ) , which are among the largest obtained with various techniques in the experiments working in the field @xcite .\\nthis result would allow to nearly double the argo - ybj sensitivity to a given source obtained from the pointing accuracy alone@xcite or , equivalently , to reduce by a factor four the time needed to observe it above the hadron background , with a given statistical significance .\\n.[tab : ann]_main characteristics of the simulated data sample ( no . of ann training events , average primary energy , .... ) together with the values of @xmath79 for @xmath12/h discrimination that resulted from this work . _ [ cols=\\\"^,^,^,^,^,^\\\",options=\\\"header \\\" , ]\\nwe presented the first results of a multiscale image analysis performed on monte carlo events by taking into account all the processes of shower development in the atmosphere and a full simulation of the argo - ybj detector response .\\nthe images have been analyzed at different length scales and their multifractal nature has been studied .\\na set of eight image parameters has been identified and used as the input for an artificial neural network , which was then trained in order to discriminate gamma initiated from proton - initiated showers .\\nsince this is the first attempt of this kind of analysis in a eas detector like argo - ybj , we decided to restrict this study to events with the core at the detector center and azimuth angles not larger than 15 degrees , while all the energies with the correct spectral dependencies have been simulated in a wide range . at this level\\nwe also neglected the contribution given by primary nuclei heavier than protons .\\nthis is a good first order approximation because of the proton - dominated cosmic ray composition in the considered energy region .\\nfurthermore heavier - nuclei - induced showers would produce event patterns with characteristics even more different from gamma - initiated ones .\\nif the results obtained in this first study will be confirmed by a further analysis on the whole event categories ( now in progress ) , the detector sensitivity to a given source would nearly double or , equivalently , the time needed to observe it above the hadron background , with a given statistical significance , would be reduced by a factor four .\\nthe best performances in @xmath12/h discrimination have been obtained for photon primary energies in the few tev range , while at higher energy this analysis might be well complemented by measuring the muon content of the shower @xcite .\\n99 c.bacci et al . , nucl .\\n& meth . in phys\\n* a443 * , 342 ( 2000 ) c.bacci et al . , astroparticle phys . *\\n17 * , 151 ( 2002 ) r.s.miller and s.westerhoff , astroparticle phys . * 11 * , 379 ( 1999 ) s.bussino and s.m.mari , astroparticle phys . * 15 * , 65 ( 2001 ) a. haungs et al .\\n, astroparticle phys .\\n* 12 * 145 ( 1999 ) b. m. sch@xmath80fer et al .\\n@xmath81 meth . in phys .\\nres . , * a465 * 342 ( 2001 ) jan w. kantelhardth et al .\\n, physica * a220 * 219 ( 1995 ) d.heck et al .\\n, report fzka 6019 , forschungszentrum , karlsruhe ( 1998 ) r.brun et al .\\n, cern publication dd / ee/84/1 ( 1992 ) s.vernetto et al . , proceedings of the 28@xmath82 icrc conference , tsukuba , 2003 .\\nsee k.fratini et al .\\n, these proceedings .\",\n          \"kolmogorov complexity ( also known as kolmogorov - chaitin or program - size complexity ) is recognized as a fundamental concept , but it is also often thought of as having little or no applicability because it is not possible to provide stable numerical approximations for finite  particularly short  strings by using the traditional approach , namely lossless compression algorithms .\\nwe advance a method that can overcome this limitation , and which , though itself limited in ways both theoretical and numerical , nonetheless offers a means of providing sensible values for the complexity of short strings , complementing the traditional lossless compression method that works well for long strings .\\nthis is done at the cost of massive numerical calculations and through the application of the coding theorem from algorithmic probability theory that relates the frequency of production of a string to its kolmogorov complexity .\\nbennett s logical depth , on the other hand , is a measure of the complexity of strings that , unlike kolmogorov complexity , measures the _ organized _ information content of a string . in\\n@xcite an application inspired by the notion of logical depth was reported in the context of the problem of image classification .\\nhowever , the results in this paper represent the first attempt to provide direct numerical approximations of logical depth .    the independence of the two measures kolmogorov complexity and logical depth which has been established theoretically , is also numerically tested and confirmed in this paper .\\nour work is in agreement with what the theory predicts , even for short strings  despite the limitations of our approach .\\nour attempt to apply these concepts to practical problems ( detailed in a series of articles ( see e.g.  @xcite ) ) is novel , and they are indeed proving to have interesting applications where evaluations of the complexity of finite short strings are needed  @xcite .    in sections\\n[ kolmochap ] ,  [ ld ] ,  [ codingchap ] and  [ formal ] , we introduce the measures , tools and formalism used for the method described in section  [ dist ] . in section  [ comparison ] ,\\nwe report the numerical results of the evaluation and analysis of the comparisons among the various measures , particularly the connection between number of instructions , integer valued program - size complexity , kolmogorov complexity approximated by means of the coding theorem method , and logical depth .\\nwhen researchers have chosen to apply the theory of algorithmic information ( ait ) , it has proven to be of great value despite initial reservations  @xcite .\\nit has been successfully applied , for example , to dna false positive repeat sequence detection in genetic sequence analysis  @xcite , in distance measures and classification methods  @xcite , and in numerous other applications  @xcite .\\nthis effort has , however , been hamstrung by the limitations of compression algorithms  currently the only method used to approximate the kolmogorov complexity of a string  given that this measure is not computable .\\ncentral to ait is the basic definition of plain algorithmic ( kolmogorov - chaitin or program - size ) complexity  @xcite :    @xmath3    where @xmath4 is a universal turing machine and @xmath5 the program that , running on @xmath4 , produces @xmath1 . traditionally , the way to approach the algorithmic complexity of a string has been by using lossless compression algorithms .\\nthe result of a lossless compression algorithm applied to @xmath1 is an upper bound of the kolmogorov complexity of @xmath1 .\\nshort strings , however , are difficult to compress in practice , and the theory does not provide a satisfactory solution to the problem of the instability of the measure for short strings .    the invariance theorem , however , guarantees that complexity values will only diverge by a constant @xmath6 ( e.g. the length of a compiler or a translation program ) . +\\n* invariance theorem * ( @xcite ) : if @xmath7 and @xmath8 are two universal turing machines , and @xmath9 and @xmath10 the algorithmic complexity of @xmath1 for @xmath7 and @xmath8 respectively , there exists a constant @xmath6 such that :    latexmath:[\\\\[\\\\label{invariance }     hence the longer the string , the less important @xmath6 is ( i.e. the choice of programming language or universal turing machine ) .\\nhowever , in practice @xmath6 can be arbitrarily large , thus having a very great impact on the stability of kolmogorov complexity approximations for short strings .\\na measure of the structural complexity ( i.e. richness of structure and organization ) of a string can be arrived at by combining the notions of algorithmic information and time complexity . according to the concept of logical depth @xcite , the complexity of a string\\nis best defined by the time that an unfolding process takes to reproduce the string from its shortest description .\\nwhile kolmogorov complexity is related to compression length , bennett s logical depth is related to decompression time .    a typical example that illustrates the concept of logical depth , underscoring its potential as a measure of complexity , is a sequence of fair coin tosses .\\nsuch a sequence would have a high information content ( kolmogorov complexity ) because the outcomes are random , but it would have no structure because it is easily generated . the string 1111  1111 would be equally shallow as measured by logical depth . its compressed version , while very small , requires little time to decompress into the original string . in contrast , the binary expansion of the mathematical constant @xmath12 is not shallow , because though highly compressible and hence having a low kolmogorov complexity , it requires non - negligible computational time to produce arbitrary numbers of digits from its shortest program ( or indeed from any short program computing the digits of @xmath12 ) .\\na detailed explanation pointing out the convenience of the concept of logical depth as a measure of organized complexity as compared to plain algorithmic complexity , which is what is usually used , is provided in @xcite . for finite strings , one of bennett s formal approaches to the logical depth of a string\\nis defined as follows : + let @xmath1 be a string and @xmath13 a significance parameter . a string s\\ndepth at significance @xmath13 is given by @xmath14    with @xmath15 the length of the shortest program for @xmath1 , ( therefore @xmath16 ) .\\nin other words , @xmath17 is the least time @xmath18 required to compute @xmath1 from a @xmath13-incompressible program @xmath5 on a universal turing machine @xmath4 .\\neach of the three linked definitions of logical depth provided in  @xcite comes closer to a definition in which near - shortest programs are taken into consideration . in this experimental approach\\nwe make no such distinction among significance parameters , so we will denote the logical depth of a string @xmath1 simply by @xmath2 .    like @xmath16 ,\\n@xmath2 as a function of @xmath1 is uncomputable .\\na novel feature of this research is that we provide exact numerical approximations for both measures @xmath16 and @xmath2 for specific short @xmath1 , allowing a direct comparison .\\nthis was achieved by running a large set of random turing machines and finding the smallest and fastest machines generating each output string .\\nhence these approximations are deeply related to another important measure of algorithmic information theory .\\nthe algorithmic probability ( also known as levin s semi - measure ) of a string @xmath1 , is a measure that describes the expected probability of a random program @xmath5 running on a universal ( prefix - free  . for details see  @xcite ) . ] ) turing machine @xmath4 producing @xmath1 .\\nformally  @xcite ,    @xmath19    i.e. the sum over all the programs for which @xmath4 with @xmath5 outputs @xmath1 and halts .\\nlevin s semi - measure @xmath20 defines a distribution known as the _ universal distribution _  @xcite .\\nit is important to notice that the value of @xmath20 is dominated by the length of the smallest program @xmath5 ( when the denominator of @xmath21 reaches its largest value ) .\\nthe length of the smallest program @xmath5 that produces the string @xmath1 is @xmath16 .\\nthe semi - measure @xmath20 is therefore also uncomputable , because for every @xmath1 , @xmath20 requires the calculation of @xmath22 , involving @xmath23 , which is itself uncomputable . an extension of @xmath20 to non - binary alphabets\\nis natural .\\nmore formally , @xmath24 can be associated with the original definition for binary strings .\\nhowever , one may want to extend @xmath24 to @xmath25 , in which case for every @xmath26 , the function @xmath27 is semi - computable ( for the same reason that @xmath28 is uncomputable ) .\\nan alternative  @xcite to the traditional use of compression algorithms to approximate @xmath23 can be derived from a fundamental theorem that establishes the exact connection between @xmath20 and @xmath16 . +\\n* coding theorem * ( levin  @xcite ) : @xmath29    where we will use @xmath0 to indicate that @xmath23 has been approximated by means of @xmath21 through the coding theorem .\\nan important property of @xmath21 as a semi - measure is that it dominates any other effective semi - measure @xmath30 , because there is a constant @xmath31 such that for all @xmath1 , @xmath32 .\\nfor this reason @xmath20 is often called a _ universal distribution _  @xcite .\\nthe ability of a universal turing machine to simulate any algorithmic process has motivated and justified the use of universal turing machines as the language framework within which definitions and properties of mathematical objects are given and studied .    however , it is important to describe the formalism of a turing machine , because exact values of algorithmic probability for short strings will be approximated under this model , both for @xmath16 through @xmath20 ( denoted by @xmath0 ) , and for @xmath16in terms of the number of instructions used by the smallest turing machine producing @xmath1 .\\n+ consider a turing machine @xmath33 with alphabet @xmath34 symbols , @xmath35 states and an additional halting state denoted by @xmath36 ( as defined by rado in his original busy beaver paper  @xcite ) . at the outset\\nthe turing machine is in its initial state @xmath37 .\\nthe machine runs on a @xmath38-way unbounded tape .\\nits behavior is determined by the transition function @xmath39 .\\nso , at each step :    the machine s current `` state '' ( instruction ) @xmath1 ; and    the tape symbol the machine s head is scanning @xmath26    define the transition @xmath40 with    a unique symbol @xmath41 to write ( the machine can overwrite a @xmath37 on a @xmath36 , a @xmath36 on a @xmath37 , a @xmath37 on a @xmath37 , and a @xmath36 on a @xmath36 ) ;    a direction @xmath13 to move in : @xmath42 ( left ) , @xmath37 ( right ) or @xmath36 ( none , when halting ) ; and    a state @xmath43 to transition into ( which may be the same as the one it was in ) .\\nthe machine halts if and when it reaches the special halt state @xmath36\\n. there are @xmath44 turing machines with @xmath45 states and @xmath38 symbols according to the formalism described above , as there are @xmath46 entries in the transition table and any of them may have @xmath47 possible instructions : there are @xmath38 halting instructions ( writing ` 0 ' and ` 1 ' ) and @xmath48 non - halting instructions ( @xmath38 movements , @xmath38 possible symbols to write and @xmath45 states ) .\\nthe output string is taken from the number of contiguous cells on the tape the head of the halting @xmath45-state machine has gone through .\\na turing machine is considered to produce an output string only if it halts .\\nthe output is what the machine has written on the tape .\\nin order to arrive at an approximation of @xmath20 , a method based on the coding theorem was advanced in  @xcite .\\nit is captured in the following function .\\nlet @xmath33 be a turing machine in @xmath49 with empty input . then :    @xmath50    where @xmath51 denotes the number of elements of @xmath52 .\\nlet @xmath21 be fixed .\\nit has been proved  @xcite that the function @xmath53 is non - computable ( due to the denominator ) .\\nhowever , @xmath54 for fixed and small values @xmath45 and @xmath21 is computable for values of the busy beaver problem  @xcite that are known . for @xmath55 , for example , the busy beaver function @xmath56 tells us that @xmath57  @xcite , so given a turing machine with 4 states running on a blank tape that hasnt halted after 107 steps , we know it will never stop .    more generally , for every string @xmath1 ( with alphabet @xmath58 ) one can compute a sequence @xmath59 which converges to @xmath60 when @xmath61 . for @xmath62\\nwe compute for @xmath18 steps all @xmath21-turing machines with @xmath45 states ( there is a finite number of them ) and compute the quotient for @xmath60 for machines that halted before @xmath18 steps . since @xmath62 converges for every @xmath1 to @xmath63 (\\n@xmath21 fixed , @xmath45 fixed , @xmath1 fixed ) , the value of @xmath64 converges for fixed @xmath21 and @xmath45 . in this specific sense @xmath65\\nis approachable , even if @xmath62 as a function of increasing time @xmath18 may increase when a machine produces @xmath1 , or decrease when a machine halts without producing @xmath1 . by the invariance theorem ( eq .  [ invariance ] ) and the coding theorem ( eq .  [ coding ] ) , @xmath63 is guaranteed to converge to @xmath20 .\\nexact values of @xmath62 were previously calculated  @xcite for @xmath66 symbols and @xmath67 states for which the busy beaver values are known .\\nthat is , a total of 36 , 10000 , 7529536 and 11019960576 turing machines respectively .\\nthe distributions were very stable and are proving to be capable of delivering applications that are also in agreement with results from lossless compression algorithms  @xcite for boundary cases ( where both methods can be applied ) , hence validating the utility of both methods ( compression being largely validated by its plethora of applications and by the fact that it achieves an approximation of @xmath23 , see e.g.  @xcite ) .\\nthe chief advantage of the coding theorem method , however , is that it is capable of dealing with short entities ( unlike compression algorithms , which are designed for large entities ) .\\nthere are 26559922791424 turing machines with 5 states and 2 symbols , and the values of busy beaver functions for these machines are unknown . in what follows\\nwe describe how we proceeded .\\ncalculating @xmath68 is an improvement on our previous numerical evaluations and provides a larger data set to work with , allowing us to draw more significant statistical conclusions vis -  - vis the relation between these calculations and strict integer value program - size complexity , as well as to make a direct comparison to bennett s logical depth .\\nwe did not run all the turing machines with 5 states to produce @xmath69 , because one can take advantage of symmetries and anticipate some of the behavior of the turing machines directly from their transition tables without actually running them ( this is impossible generally due to the halting problem , but some reductions are possible ) .\\nif @xmath70 is the set of turing machines with @xmath45 states and @xmath21 symbols , as defined above , we reduce it to : @xmath71 where @xmath39 is the transition function of @xmath33 .\\nso @xmath72 is a subset of @xmath70 , with machines with the transition corresponding to initial state @xmath37 and symbol @xmath36 ( this is the initial transition in a ` 0'-filled blank tape ) moving to the right and changing to a state different from the initial and halting ones .    for machines with two symbols , @xmath73 as there are @xmath74 different initial transitions ( the machine can write ` 0 ' or ` 1 ' and move to one of the @xmath75 states in @xmath76 ) , and for the other @xmath77 transitions there are @xmath47 possibilities , as in @xmath70 .\\nafter running @xmath78 on a ` 0'-filled tape , the procedure for completing the output strings so that they reach the frequency they have in @xmath70 is :    * for every @xmath79 , * * if @xmath33 halts and produces the output string @xmath1 , add one occurrence of @xmath80 , the reverse of @xmath1 .\\n* * if @xmath33 does not halt , count another non - halting machine .\\n+ these two completions add the output ( or number of non - halting machines ) of @xmath81 new machines , one for each machine in @xmath78 .\\nthese new machines are left - right symmetric to the machines in @xmath78 .\\nformally , this is the set @xmath82 when the original machine halts , its symmetric counterpart halts too and produces the reversed strings , and if the original machine does not halt , neither does the symmetric machine . this way we consider the output of all machines with the initial transition moving to the left and to a state not in @xmath83 . *\\ninclude @xmath84 occurrences of string `` 1 '' .\\nthis corresponds to the machines writing ` 1 ' and halting at the initial transition .\\nthere is just one possible initial transition for these machines ( move to the halting state , write ` 1 ' and remain in the initial cell ) .\\nthe other @xmath77 transitions can have any of the @xmath47 possible instructions .\\n* include @xmath84 occurrences of string `` 0 '' .\\nthis is justified as above , for machines writing ` 0 ' and halting at the initial transition . *\\ninclude @xmath85 additional non - halting machines , corresponding to machines remaining in the initial state in the initial transition ( these machines never halt , as they remain forever in the initial state ) .\\nthere are @xmath86 initial transitions of this kind , as the machine can write @xmath38 different symbols and move in @xmath38 possible directions .\\nif we sum @xmath87 and the machines considered above , having completed them in the manner described , we get the output corresponding to the @xmath88 machines in @xmath89 .\\nmoreover , we need the output of those machines starting with a ` 0'-filled tape and with a ` 1'-filled tape .\\nbut we do not run any machine twice , as for every machine @xmath90 producing the binary string @xmath1 starting with a ` 1'-filled tape , there is also a 0 - 1 symmetric machine @xmath91 ( where the role of 1 ( of 0 ) in the transition table of @xmath92 is the role of 0 ( of 1 ) in the transition table of @xmath33 ) that when starting with a ` 0'-filled tape produces the complement to one of @xmath1 , that is , the result of replacing all 0s in s with 1s and all 1s with 0s .\\nso we add the complement to every one of the strings found and count the non - halting machines twice to obtain the output of all machines in @xmath89 starting both with a ` 0'-filled tape and with a ` 1'-filled tape .\\nto construct @xmath69 , we ran the @xmath93 machines in @xmath94 , which is @xmath95 of @xmath96 .\\nthe output strings found in @xmath94 , together with their frequencies , were completed prior to constructing @xmath69 , following the procedure explained above .\\nit is useful to avoid running machines that we can easily determine will not stop .\\nthese machines will consume the runtime without yielding an output . as we have shown above\\n, we can avoid generating many non - halting machines .\\nin other cases , we can detect them at runtime , by setting appropriate filters .\\nthe theoretical limit of the filters is the halting problem , which means that they can not be exhaustive . but a practical limit is imposed by the difficulty of checking some filters , which takes up more time than the runtime that is saved .\\nwe have employed some filters that have proven useful .\\nbriefly , these are :    * * machines without transitions to the halting state*. while the transition table is being filled , the simulator checks to ascertain whether there is some transition to the halting state .\\nif not , it avoids running it .\\n* * escapees*. these are machines that at some stage begin running forever in the same direction . as they are always reading new blank symbols , as soon as the number @xmath6 of non - previously visited positions is greater than the number @xmath45 of states , we know that they will not stop , because the machines have necessarily entered an infinite loop . given that @xmath97 , while visiting the last @xmath6 new cells , some of the @xmath45 states have been repeated , and will repeat forever , as the machine s behavior is deterministic .\\n* * cycles of period two*. these cycles are easy to detect .\\nthey are produced when in steps @xmath18 and @xmath98 the tape is identical and the machine is in the same state and the same position .\\nwhen this is the case , the cycle will be repeated infinitely .\\nthese filters were implemented in our c++ simulator , which also uses the reduced enumeration of section  [ sec : some - reductions ] . to test them we calculated @xmath99 with the simulator and compared the output to the list that was computed in  @xcite , arriving at exactly the same results , and thereby validating our reduction techniques .\\nrunning @xmath100 without reducing the enumeration or detecting non - halting machines took 952 minutes . running the reduced enumeration with non - halting detectors took 226 minutes .\\nthe busy beaver for turing machines with 4 states is known to be 107 steps @xcite , that is , any turing machine with 2 symbols and 4 states running longer than 107 steps will never halt .\\nhowever , the exact number is not known for turing machines with 2 symbols and 5 states , although it is believed to be 47176870 , as there is a candidate machine that runs for this length of time and halts and no machine with a greater runtime has yet been found .\\nso we decided to let the machines with 5 states run for 4.6 times the busy beaver value for 4-state turing machines ( for 107 steps ) , knowing that this would constitute a sample significant enough to capture the behavior of turing machines with 5 states .\\nthe chosen runtime was rounded to 500 steps , which was used to construct the output frequency distribution for @xmath69 .\\nnot all 5-state turing machines have been used to build @xmath69 , since only the output of machines that halted at or before 500 steps was taken into consideration . as an experiment to ascertain how many machines we were leaving out , we ran @xmath101 random turing machines for up to 5000 steps . among these ,\\nonly 50 machines halted after 500 steps and before 5000 ( that is , a fraction less than @xmath102 , because in the reduced enumeration we do nt include those machines that halt in one step or that we know wo nt halt before we generate them , so it s a smaller fraction ) , with the remaining 1496491379 machines not halting at 5000 steps . as far as these are concerned  and given that the busy beaver values for 5 states are unknown  we do not know after how many steps they would eventually halt , if they ever do . according to the following analysis , our election of a runtime of 500 steps therefore provides a good estimation of @xmath69 .\\nthe frequency of runtimes of ( halting ) turing machines has theoretically been proven to drop exponentially @xcite , and our experiments are closer to the theoretically predicted behavior . to estimate the fraction of halting machines that were missed because turing machines with 5 states were stopped after 500 steps , we hypothesize that the number of steps @xmath56 a random halting machine needs before halting is an exponential random variable , defined by @xmath103 we do not have direct access to an evaluation of @xmath104 , since we only have data for those machines for which @xmath105 .\\nbut we may compute an approximation of @xmath106 , @xmath107 , which is proportional to the desired distribution .    a non - linear regression using ordinary least - squares\\ngives the approximation @xmath108 with @xmath109 and @xmath110 .\\nthe residual sum - of - squares is @xmath111 ; the number of iterations with starting values @xmath112 and @xmath113 is nine .\\nthe model s @xmath114 is the same @xmath114 appearing in the general law @xmath104 , and may be used to estimate the number of machines we lose by using a 500 step cut - off point for running time : @xmath115 .\\nthis estimate is far below the point where it could seriously impair our results : the less probable ( non - impossible ) string according to @xmath69 has an observed probability of @xmath116 .\\nalthough this is only an estimate , it suggests that missed machines are few enough to be considered negligible .\\nwe now study the relation of @xmath0 to the minimal number of instructions used by a turing machine producing a given string , and to bennett s concept of logical depth .\\nas expected , @xmath0 shows a correlation with the number of instructions used but not with logical depth .\\nfirst , we are interested in the relation of @xmath117 to the minimal number of instructions that a turing machine producing a string @xmath1 uses .\\nmachines in @xmath69 have a transition table with 10 entries , corresponding to the different pairs @xmath49 , with @xmath1 one of the five states and @xmath21 either `` 0 '' or `` 1 '' .\\nthese are the 10 instructions that the machine can use .\\nbut for a fixed input not all instructions are necessarily used .\\nthen , for a blank tape , not all machines that halt use the same number of instructions .\\nthe simplest cases are machines halting in just one step , that is , machines whose transition for @xmath118 goes to the halting state , producing a string `` 0 '' or `` 1 '' .\\nso the simplest strings produced in @xmath68 are computed by machines using just one instruction .\\nwe expected a correlation between the @xmath0-complexity of the strings and the number of instructions used . as we show\\n, the following experiment confirmed this .\\nwe used a sample of @xmath119 random machines in the reduced enumeration for @xmath68 , that is , @xmath120 the total number of machines .\\nthe output of the sample returns the strings produced by halting machines together with the number of instructions used , the runtime and the instructions for the turing machine ( see fig .  [\\nfig : distribins ] ) . in order to save space\\n, we only saved the smallest number of instructions found for each string produced , and the smallest runtime corresponding to that particular number of instructions .\\nvalues according to the minimum number of instructions required . each \\ndrop - like \\\" distribution is the set of strings that are minimally produced with the same number of instructions ( horizontal axis ) .\\nthe more instructions needed to produce the strings , the more complex they are ( vertical axis in @xmath121 units).,width=377 ]    after doing the appropriate symmetry completions we have 99584 different strings , which is to say almost all the 99608 strings found in @xmath68 .\\nthe number of instructions used goes from 1 to 10 .\\nwhen 1 instruction is used only `` 0 '' and `` 1 '' are generated , with a @xmath0 value of @xmath122 . with 2 instructions ,\\nall 2-bit strings are generated , with a @xmath0 value of @xmath123 . for 3 or more instructions , fig .\\n[ fig : distribins ] shows the distribution of values of @xmath0 .\\ntable  [ tab : meankl ] shows the mean @xmath0 values for the different numbers of instructions used .\\n.mean @xmath0 and string length for different numbers of instructions used .\\n[ cols=\\\">,>,>\\\",options=\\\"header \\\" , ]     we now provide some examples of the discordance between @xmath0 and @xmath124 .\\n`` 0011110001011 '' is a string with high @xmath0 and low @xmath124 .\\n[ fig : tablelowtime ] shows the transition table of the smallest machine found producing this string .\\nthe runtime is low  just 29 steps ( of the 99584 different strings found in our sample , only 3360 are produced in fewer steps ) , but it uses 10 instructions and produces a string with complexity @xmath125 .\\nit is the greatest complexity we have calculated for @xmath0 .\\n[ fig : execlowtime ] shows the execution of the machine .            on the other hand , `` @xmath126 ''\\nis a string with high @xmath124 but a low @xmath0 value .\\n[ tmtablehightime ] shows the transition table of the machine found producing this string , and fig .\\n[ tmrunhightime ] depicts the execution .\\nthe machine uses 9 instructions and runs for 441 steps ( only 710 strings out of the 99584 strings in our sample require more time ) but its @xmath0 value is @xmath127 .\\nthis is a low complexity if we consider that in @xmath0 there are 99608 strings and that 90842 are more complex than this one .    ' ' .,width=415 ]    '' . with a high runtime ,\\nit produces a string with low complexity.,width=453 ]    we may rate the overall strength of the relation between @xmath128 and @xmath124 by the correlation @xmath129 , corresponding to a medium positive link . as we previously mentioned , however , the fact that the length @xmath130 of the strings is linked with both variables may bias our interpretation .\\na more relevant measure is thus @xmath131 , a slight negative but with no significant value between @xmath128 and @xmath124 once @xmath130 is controlled .\\nthe results in this paper are important because these measures can be better studied and understood under a specific but widely known general formalism .\\nwhat we have found is very interesting because it is what one would wish in the best case scenario , stable and reasonable distributions rather than chaotic and unstable ones .\\nthe results also suggest that these measures can be applied even if numerically approximated using a specific model of computation .\\nfor example , as we expected , the kolmogorov - chaitin complexity evaluated by means of levin s coding theorem from the output distribution of small turing machines correlates with the number of instructions used but not with logical depth .\\nlogical depth also yields a measure that is different from the measure obtained by considering algorithmic complexity ( @xmath23 ) alone , and this investigation proves that all these three measures ( kolmogorov - chaitin complexity , solomonoff - levin algorithmic probability and bennett s logic depth ) are consistent with theoretical expectations .\\n@xmath23 as a measure of program - size complexity is traditionally expected to be an integer ( the length of a program in bits ) , but when evaluated through algorithmic probability using the coding theorem it retrieves non - integer values ( still bits ) .\\nthese results confirm the utility of non - integer values in the approximation of the algorithmic complexity of short strings , as they provide finer values with which one can tell apart small differences among short strings  which also means one can avoid the longer calculations that would be necessary in order to tell apart the complexity of very small objects if only integer values were allowed .\\nthus it also constitutes a complementary and alternative method to compression algorithms .    an _ on - line algorithmic complexity calculator _ ( or oacc ) is now available at http://www.complexitycalculator.com .\\nit represents a long - term project to develop an encompassing universal tool implementing some of the measures and techniques described in this paper .\\nit is expected to be expanded in the future as it currently only implements numerical approximations of kolmogorov complexity and levin s semi - measure for short binary strings .\\nmore measures , more data and better approximations will be gradually incorporated in the future , covering a wider range of objects , such as longer binary strings , non - binary strings and multidimensional arrays ( such as images ) .\\nbennett , logical depth and physical complexity in rolf herken ( ed ) _ the universal turing machine  a half - century survey , _ oxford university press 227257 , 1988 .\\nbennett , how to define complexity in physics and why . in _ complexity , entropy and the physics of information .\\n_ zurek , w. h. , addison - wesley , eds .\\nsfi studies in the sciences of complexity , p 137 - 148 , 1990 .\\nbrady , the determination of the value of rado s noncomputable function @xmath132 for four - state turing machines , _ mathematics of computation 40 _ ( 162 ) : 647665 , 1983 .\\ncalude , _ information and randomness _ , springer , 2002 .\\ncalude and m.a .\\nstay , most programs stop quickly or never halt , _ advances in applied mathematics _\\n, 40 , 295 - 308 , 2008 .\\nchaitin , on the length of programs for computing finite binary sequences : statistical considerations , _ journal of the acm _ , 16(1):145159 , 1969 .\\n_ from philosophy to program size , _ 8th .\\nestonian winter school in computer science , institute of cybernetics , tallinn , 2003 .\\nr. cilibrasi , p. vitanyi , clustering by compression , _ ieee transactions on information theory , _ 51 , 4 , 15231545 , 2005 .\\nt.m . cover and j.a .\\nthomas , _ information theory , _\\nj. wiley and sons , 2006 .\\ndelahaye , _ complexit alatoire et complexit organise , _ editions quae , 2009 .\\ndelahaye , h. zenil , towards a stable definition of kolmogorov - chaitin complexity , arxiv:0804.3459 , 2007 .\\ndelahaye and h. zenil , on the kolmogorov - chaitin complexity for short sequences . in c.\\ncalude ( ed . ) , _ randomness and complexity : from leibniz to chaitin _ , world scientific , 2007 .\\ndelahaye & h. zenil , numerical evaluation of the complexity of short strings : a glance into the innermost structure of algorithmic randomness , _ applied math . and comp .\\nw. kircher , m. li , and p. vitanyi , the miraculous universal distribution , _ the mathematical intelligencer , _ 19:4 , 715 , 1997 .\\nkolmogorov , three approaches to the quantitative definition of information , _ problems of information and transmission _ , 1(1):17 , 1965 . l. levin , laws of information conservation ( non - growth ) and aspects of the foundation of probability theory .\\n, _ problems in form . transmission _ 10 . 206210 , 1974 . m. li , p. vitnyi , _ an introduction to kolmogorov complexity and its applications , _ springer , 2008 .\\n . rivals , m. dauchet , j .-\\ndelahaye , o. delgrange , compression and genetic sequence analysis . , _ biochimie _ , 78 , pp 315 - 322 , 1996 . t. rad , on non - computable functions , _ bell system technical journal , _ vol .\\n3 , pp . 877884 , 1962 .\\nsolomonoff , a formal theory of inductive inference : parts 1 and 2 . _ information and control _ , 7:122 and 224254 , 1964 .\\nh. zenil , une approche exprimentale  la thorie algorithmique de la complexit , dissertation in fulfilment of the degree of doctor in computer science ( jury members : j .- p .\\ndelahaye and c.s .\\ncalude , g. chaitin , s. grigorieff , p. mathieu and h. zwirn ) , universit de lille 1 , 2011 .\\nh. zenil , f. soler - toscano , j .-\\ndelahaye and n. gauvrit , two - dimensional kolmogorov complexity and validation of the coding theorem method by compressibility , arxiv:1212.6745 [ cs.cc ] .\\nh. zenil , j .-\\ndelahaye and c. gaucherel , image information content characterization and classification by physical complexity , _ complexity _ , vol .\\n173 , pages 2642 , 2012 .\",\n          \"@xmath0 compact muon solenoid ( cms ) @xcite is one of the two general - purpose major detectors for the lhc accelerator .\\nthe main goal of this experiment is a precise test of the standard model and observation of the higgs boson as well as the search for diverse signatures of new physics .\\nit will be achieved by identifying and precisely measuring muons , electrons and photons over a large energy range , by determining the signatures of quarks and gluons through the measurement of jets of charged and neutral particles ( hadrons ) with moderate precision , and by measuring the missing transverse energy flow , which will enable the signatures of non - interacting new particles as well as neutrinos to be identified .\\n@xmath0 the standard model higgs boson with a mass between 95 and 150 @xmath1 would be discovered via its two - photon decay , with a mass between 135 and 525 @xmath1  in the four - lepton channel .\\ntagging the events produced by ww- and zz - fusion by detecting the characteristic forward jets , and using decay modes with larger branching ratios ( h @xmath2 ww @xmath2 lvjj , and h @xmath2 zz @xmath2 lljj ) should allow the discovery range for sm higgs boson to be extended up to 1 tev .\\nthe two - photon and four - lepton channels are also crucial for the discovery of the higgs boson in the minimal supersymmetric standard model\\n. events with many high energy jets and large missing transverse energy are the obvious and model independent signature in searches for the supersymmetric partners of quarks and gluons .    @xmath0the cms calorimeters will play a significant role in exploiting the physics potential offered by lhc .\\nthe main functions are to identify and measure precisely the energy of photons and electrons , to measure the energy of jets , and to provide hermetic coverage for measuring the missing transverse energy .\\nin addition , good efficiency for electron and photon identification as well as excellent background rejection against hadrons and jets are required .\\n@xmath0 one of the principal cms objectives is to construct a very high performance electromagnetic calorimeter ( ecal ) .\\na homogeneous scintillating crystal calorimeter with high granularity gives the best performance for energy resolution enhancing the h @xmath3 discovery potential at the initially lower luminosities .\\nthe choice of crystals for ecal , as well as the thinness of the barrel calorimetry , imposes severe constraints on the hadron calorimeter ( hcal ) design and tempers its performance .\\nin particular , the cms calorimeter system is strongly non - compensating .\\nthis leads to a non - gaussian energy distribution and non - linear response in energy to hadrons .\\n@xmath0 in order to ensure a gaussian response and good linearity of the calorimetry systems , different weighting methods for reconstruction of the hadron energy , which have the effect of simultaneously minimizing the non - linearity and the energy resolution can be employed .\\n@xmath0the present paper is devoted to the application of a feed - forward neural network for reconstruction of the energy deposited in the cms calorimeter system by hadrons and jets .\\na significant improvement of the energy resolution and linearity in comparison with other weighting methods has been achieved .\\n@xmath0the paper is organized as follows : in section 2 a brief description of the cms and its calorimeter system is presented . in section 3 ,\\nsome aspects of development of hadron showers in the sampling calorimeters are considered and two of the conventional weighting methods for reconstruction of the pion and jet energy in the case of cms are applied .\\na basic information about neural networks and their usage for processing signals from calorimetry systems is given in section 4 . in section 5 , details of our approach to the application of neural networks for energy reconstruction and the particular feed - forward network used in the case of cms detector are described .\\nthe results obtained are presented in section 6 .\\n@xmath0 the cms @xcite detector has an overall length of 21.6 m , with a calorimeter coverage up to a pseudorapidity of @xmath4 , a radius of 7.5 m , and a total weight of about 12500 t ( see figure 1.cms ) .\\ncms consists of a powerful inner tracking system , a scintillating crystal electromagnetic calorimeter followed by a sampling hadron calorimeter mounted inside the cryostat vessel of the 4 t solenoidal superconducting magnet , 13 m long with an inner diameter of 5.9 m. the magnet is surrounded by 5 `` wheels '' ( cylindrical structure ) and 2 endcaps ( disks ) of muon absorber and muon tracking chambers composing the muon detector system .\\n@xmath0 the cms tracker consists of a silicon pixel barrel and forward disks , followed by silicon microstrip devices placed in a barrel and forward disk configuration .\\nthe tracker is located inside the calorimeter system and is supported by it .\\n@xmath0 the electromagnetic calorimeter @xcite comprises a barrel ( @xmath5 ) and two endcaps .\\nthe active medium is made out of scintillating lead tungstate crystals ( @xmath6 ) .\\nthe choice is based on the following properties of these crystals : a short radiation length of 0.89 cm , a small moliere radius of 2.19 cm , the scintillating process is fast  85 @xmath7 of the light is emitted in 20 ns .\\nthe transverse granularity of @xmath8 x @xmath9= @xmath10 x @xmath10 . corresponds to a crystal front face of about 22 x 22 @xmath11 . in the endcaps @xmath12 ,\\nthe granularity increases progressively to a maximum value of @xmath8 x @xmath9= @xmath13 x @xmath13 .\\nthe total thickness of about 26 radiation lengths , corresponding to a crystal length 23 cm is enough to limit the longitudinal shower leakage to an acceptable level .\\nthe presence of a preshower ( 3 @xmath14 of lead ) in the endcap region allows the use of slightly shorter crystals ( 22 cm ) .\\nthe light produced by an incident particle is detected by avalanche photodiodes in the barrel and by vacuum phototriodes in the endcap .\\n@xmath0 the cms central hadron calorimeter @xcite , also consists of barrel ( hb ) and endcaps ( he ) parts , covering the central rapidity range ( @xmath15 ) .\\nboth the barrel and endcap calorimeters experience the 4 tesla field of the cms solenoid and hence are necessarily fashioned out of non - magnetic material ( copper alloy and stainless steel ) .\\nthe central hadron calorimeter is a sampling calorimeter : it consists of active material inserted between copper absorber plates .\\nthe absorber plates are 5 cm thick in the barrel and 8 cm thick in the endcap .\\nthe active elements are 4 mm thick plastic scintillator tiles read out using wavelength shifting plastic fibres .\\nthe tiles are arranged in a tower structure pointing to the interaction center .\\nthe lateral segmentation of @xmath8 x @xmath9= @xmath16 x @xmath16 has been chosen so as not to degrade di - jet mass resolution for highly - boost di - jets . only in the region near @xmath17\\nthe segment size is 0.17 x 0.17 .\\n@xmath0 the barrel hadron calorimeter is about 89 cm deep , which at @xmath18 is only 5.82 nuclear interaction lengths ( @xmath19 ) in thickness . to ensure adequate sampling depth for the entire ( @xmath15 ) region\\nthe first muon absorber layer is instrumented with scintillator tiles to form an outer hadron calorimeter ( ho ) .\\nthe two layers of scintillator of the barrel ho are divided into the same granularity as the hb and envelope the entire first layer of the muon iron absorber . in the region\\n@xmath20 additional 15 cm thick steel plates are placed in front of muon chambers . in this region ho\\nconsists of 3 sampling layers . in the endcap region ,\\nthe ho has only single sampling layers .\\n@xmath0 to extend the hermeticity of the central hadron calorimeter system up to @xmath21 , in order to have good missing transverse energy measurement , cms employs a separate forward calorimeter ( hf ) located 6 m downstream of the he endcaps .\\nthe hf calorimeter covers the region latexmath:[$3.0\\n<    a copper absorber matrix .    @xmath0 in order to minimize non - gaussian tails in the hadron energy distribution and to ensure a linear response to hadron energy , the inner barrel hadron calorimeter is divided radially ( in depth ) into two sampling hadron compartments ( hb1 and hb2 ) .\\nthere is an initial layer of sampling immediately following the ecal electronics , and 17 layers of sampling coupled together into single tower read - out .\\nthe he is also segmented into two different sampling compartments ( he1 and he2 ) . there again is initial sampling layer , followed by 18 layers joined into a single tower read - out .\\nthe ho layers form towers matching the inner hadron calorimeter granularity and are read out separately . in this way , for reconstruction of the hadron energy , in every ( @xmath23 ) tower , signals from four longitudinal read - outs ( ecal , hb1 , hb2 and ho ) are used .\\n@xmath0an important issue is the absolute calibration of the cms calorimeter system , used to determine the energy scale . for the ecal ,\\nthe individual calibration of all crystal is foreseen in two steps . in order to establish an extremely clean set of high precision initial calibration coefficients for all channels ,\\nall crystals will be scanned in the cern sps test electron beam at two energies .\\nafter installation of the calorimeter in the cms detector , an in situ calibration with physics events is envisaged .\\nthe most suitable channels for this purpose will be @xmath24 which gives energetic correlated electrons in different regions of the calorimeter .\\n@xmath25 measurements for isolated electrons also is a high - rate tool especially important at low luminosity . during the run of cms experiment\\n, a light monitoring system will track the behaviour of each channel .\\n@xmath0the hcal calibration and monitoring is designed to determine the absolute energy scale and monitor the calorimeter system for changes .\\ninitially , several individual wedges will be placed in the test beam at cern where hadrons and muons of various energies will be used to determine absolute calibration between beam energy and light yield response to the moving radioactive wire source @xcite .\\nthe absolute single hadron test beam calibration will be transferred to the rest of the calorimeter with the help of the same radioactive source .\\nthe dependence of the calorimeter response on the magnetic field @xcite makes in situ calibration using physics events obligatory . in the cms detector ,\\nsingle tracked hadrons from @xmath26-leptons , jet balancing ( @xmath27 and @xmath28 ) and di - jet resonances ( @xmath29 in top decays , @xmath30 and @xmath31 ) will be used for in situ calibrations .\\n+ fig.1 cms\\n@xmath0the reconstruction of the energy of a particle entering the calorimetry system is a nontrivial task .\\nthe difficulties are determined , first of all , by the complicity of the physical processes taking place in the development of the electromagnetic and hadron showers and the mechanism of creation of the signal in the active elements of the detectors .\\nthis leads to big fluctuations in the signals of the calorimetry system , especially in the case of hadrons ( the so called intrinsic fluctuations ) . in the case of sampling calorimeters ,\\nonly a fraction of the shower energy is dissipated in the active medium and the energy resolution is affected significantly by the fluctuations in this fraction ( sampling fluctuations ) .\\nhadron calorimeters , because the large depth required to incorporate almost the full shower , are by necessity sampling calorimeters .\\n@xmath0there is an essential difference between development of a shower initiated by electrons and photons ( electromagnetic ) and the one initiated by hadrons . in the first case ( assuming enough deep calorimeter ) high energy electron or photon incident on a calorimeter initiates a cascade of secondary electrons and photons via bremstrahlung and pair production .\\nthe multiplication continues until the energies of secondary particles fall bellow the critical energy .\\nthe further dissipation of energy is dominated by ionization and excitation .\\nthe visible energy is proportional to the energy dissipated in the calorimeter even in the case of sampling calorimeter .\\n@xmath0the mechanism for hadron shower production is quite different since it involves multiparticle production in deep inelastic hadron - nucleus collisions at high energies and thus the elementary processes are determined by the strong interactions .\\nthe large number of possible interaction processes makes the shower development more complex . the cascade is mostly composed of nucleons and pions . a significant part of pions are @xmath32 ( @xmath33 )\\n, as a result the cascade contains two distinct components namely the electromagnetic one ( @xmath32 s etc ) and the hadron one ( @xmath34 , n , etc ) .\\nthe electromagnetic component is determined essentially by the first interaction and is subject of considerable event - to - event non - gaussian fluctuations .\\nit increases with energy due to the fact that the neutral pions , developing as electromagnetic showers , do not produce any hadron interactions .\\nthe response of hadron sampling calorimeter is a function of the energy deposited in the active elements by electromagnetic component , charged hadrons , low energy neutrons and energy lost in breaking up nuclei ( invisible energy ) .\\nthe last one can be large enough and for heavy absorbers it reaches up to 40 @xmath7 of hadron part of the energy .\\nthus the ratio of response to electromagnetic and hadron showers ( @xmath35 ) is usually @xmath36 and is a function of the initial energy ( non - compensating calorimeters ) .\\nthis leads to a non - gaussian measured distribution for mono - energetic hadrons , a non - linear response in energy to hadrons and an additional contribution to the relative energy resolution ( @xmath37 ) .\\nit is possible to achieve a compensation ( @xmath38 ) by suppression of the electromagnetic signal or by boosting the non e.m .\\nhowever the requirement for precise electromagnetic calorimetry is non compatible with compensation and leads , as a rule , to strongly non - compensating calorimetry systems .\\n@xmath0another problem caused by non - compensation is the dependence of the calorimeter response on the type of the incident particle .\\nthe response for single hadrons and jets is different , because significant part of the jet energy is carried by electrons and photons .\\ndue to the different cross - sections for creation of @xmath32-mesons even for pions and protons the response of the calorimetry system is not equal .\\nthe cms calorimetry system is a typical example for strongly non - compensating hadron calorimeter .\\n@xmath0 in order to achieve a good linearity , gaussian energy distribution and the best possible energy resolution , different methods for reconstruction of the energy deposited in the calorimetry system are used .\\nthe idea is to find such calibration coefficients ( weights ) which take into account the peculiarities of the development of the hadron shower . in order to investigate the applicability of the various reconstruction techniques in the case of cms calorimetry system we have simulated the detector response to electrons and pions ( with energies 5 , 10 , 20 , 50 , 100 , 200 , 300 and 500 gev ) and single jets ( with energies 30,50,100,200,300 and 500 gev ) entering the ecal at @xmath39 and @xmath40 .\\n@xmath0 in the most common approach ( in what follows we shall refer to it as to sm ) , the energy of a single shower is given by the weighted sum of the energies deposited by it in the calorimetry system @xmath41 where the index @xmath42 corresponds to the four longitudinal compartments of the calorimeter ( ecal , hb1 , hb2 and ho ) and @xmath43 is the sum of the energies measured in the transversal towers in the @xmath44 readout .\\ngiven a large enough number of showers with known incident energy @xmath45 , the weights are determined by minimization of the width of the energy distribution with additional constraint @xmath46 .\\n@xmath0we have used the simulated pion data to determine the coefficients @xmath47 .\\nthe weights are energy dependent .\\nthis dependence is very strong for @xmath48 , which is not surprising , taking into account , that for the ecal @xmath49 .\\nif @xmath48 is fixed at the value obtained by using electron calibration , big non - linearity and non - gaussian tails in the energy distributions for pions and jets are observed , especially at low energies . the way to improve the situation is to determine @xmath48 using pions , but this requires a very clear identification in the off - line analysis of the type of the particles .\\nthe jet energy resolution evaluated by using the weights obtained for 300 gev pions ( this corresponds to calibration of the calorimeter in 300 gev pion beam , see the previous section ) is plotted on fig .\\n[ fig : res300 ] . the linearity defined as    @xmath50\\nis shown on fig .\\n[ fig : lin ] .\\nthe different electromagnetic content in the hadron showers developed by pions and jets and the non - compensation of the calorimetry system lead to the nonlinear response and induces a big constant term in the energy resolution .\\nwe have used the weights obtained for 300 gev jets ( that corresponds to in situ calibration with physical events ) to reconstruct the energy . as a result\\nthe linearity has been improved slightly ( see fig .  [\\nfig : lin ] ) and the constant term of the energy resolution has been reduced significantly ( fig .\\n[ fig : res300 ] ) .\\nhowever this step does not solve the problem with non - gaussian tails in the energy distributions and non linear response of the calorimeter .\\nwhen we use energy dependent weights for reconstruction of the energy , the linearity is restored but the energy resolution does not change ( fig .  [ fig : smh1fitr ] ) .\\n@xmath0 in the sm method the weights are sensible to the average of the fluctuations in the development of the hadron shower . when applied to individual events\\n, they always involve large errors .\\na better approach is to use a method where a different correction factor is applied to each event , depending on its nature .\\nseveral weighting techniques have been developed @xcite with the idea to suppress the signal from the electromagnetic component of hadron shower ( the so called h1-technique ) .\\nthe h1-technique we apply is of the following form : @xmath51 where @xmath52 is the energy deposited in the jth transversal tower of the ith longitudinal read - out .\\nthe weights @xmath53 and @xmath54 have been determined in the same way like in the sm approach .\\nthe results obtained are plotted on fig .\\n[ fig : res300 ] , fig .\\n[ fig : lin ] and fig .\\n[ fig : smh1fitr ] . the energy resolution is slightly improved mainly due to the reduction of the constant term .\\nhowever the nonlinearity is still too big , especially at lower energies .\\nobviously within these methods it is not possible to achieve good linearity and gaussian response .\\n@xmath0to ensure the best possible measurement of the energy , when dealing with noncompensating calorimeters , the following requirements should be satisfied :    @xmath0 to every individual event , different correction factor should be applied ( due to the big fluctuations of the hadron shower development ) ;    @xmath0 using the lateral and longitudinal energy distribution ( the only available information from the calorimeter ) , the amount of the energy dissipated in the calorimeter by electromagnetic showers ( electromagnetic part of the hadron shower ) should be estimated ;    @xmath0 the type of the particle ( electron / photon , hadron or jet ) should be determined .\\n@xmath0of course the spatial energy distribution does not gives the full information about the development of the hadron shower , however it is possible taking into account correlations between signals to determine the type of the initiator of the shower , and to estimate roughly the energy of the electromagnetic part of the shower .\\nfor example , using the longitudinal distribution of the energy , it is possible to separate electrons ( big signal in the ecal and almost nothing in the hcal ) from pions and jets .\\nthere is a significant difference in the lateral radius of the electromagnetic and hadron showers  the radius of the hadron shower is much bigger then the electromagnetic one .\\nthis feature can be used for estimation of the electromagnetic content of the shower . in some sense\\nthe h1-technique accounts for this difference and manifests better energy resolution and linearity .\\n@xmath0to solve the problem of energy reconstruction we need a method which is able to deal with many parameters , is sensitive to correlations between them and is flexible enough to react to fluctuations in the development of the hadron shower .\\none possible solution is the application of neural networks , which have proved their efficiency in such a complicated environment .\\n@xmath0 the neural networks ( nn ) are a powerful tool , which can be used for feature extraction , association , optimization , function fitting , and modeling .\\nthey have found numerous applications @xcite in the high energy physics , such as classification of particles and final states , track reconstruction , cluster trigger , particle identification , reconstruction of invariant masses and other off - line analysis @xcite .\\n@xmath0 the neural network is a system composed of many simple processing elements operating in parallel whose function is determined by network structure , connection strengths , and the processing performed at computing elements or nodes .\\nthe basic computing unit of nn is a neuron . a multi - layer feed forward ( mlff ) network consists of a set of input neurons , one or more layers of hidden neurons , and a set of output neurons .\\nthe neurons of each layer are connected to the ones in the subsequent layer .\\n@xmath0the neurons ( fig .\\n1.nn ) perform calculations in three steps via their input i , activation a and output o functions .\\nusually they are chosen in the following form : @xmath55 where @xmath56 are the connections weights , @xmath57 is some threshold . in most of the architectures\\na bias @xmath58 is implemented as an incoming link connected to a constant value 1 .\\nthe weight of the connection is trained like all other weights .\\n@xmath0 mlff networks are , usually , trained using the backpropagation learning algorithm @xcite .\\nit includes three steps : presentation of a pattern to the network , comparison of the desired output with the actual network output , backwards calculation of the error and adjustment of the weights ( wij ) .\\nthis is done by minimization of the error function , @xmath59 where @xmath60 is the output of the net and @xmath61 is the corresponding element of teaching sample .\\nbackpropagation algorithm uses an updating rule :    @xmath62 ,    where @xmath63 is the learning rate .\\nthe optimal value for @xmath63 varies significantly during the learning of the nn .\\nfor that reason we have chosen another learning algorithm  rprop @xcite .\\nit uses an individual learning rate for each weight combined with the manhattan updating rule @xcite :    @xmath64\\\\ ] ]    at every step , @xmath63 is adjusted as :    @xmath65    @xmath66    @xmath67    @xmath0 there are two possible approaches to the problem of the energy reconstruction with the assistance of nn .\\nthe first one is to use nn directly to determine the energy dissipated in the calorimetry system .\\nsuch an approach has been applied in the case of gamma - ray energy determination with gilda imaging silicon calorimeter @xcite .\\nthe energy reconstruction is performed in two steps . in the first stage ,\\na net performs a rough classification of the gamma energies in six groups .\\nthen , for each group , a dedicated net proceeds to discriminate among the different energy values . at the second stage\\neach net has an output layer , which consists of ni nodes , equal to the number of the different energy subclasses used in the training phase .\\nthe net has a discrete output and is able to classify only those energies used during the training .\\nsince gamma rays have a continuous energy distribution , the following weighted average over the values of output neurons is performed :    @xmath68    where @xmath69 is the energy corresponding to gamma rays of class @xmath70 and @xmath71 is the activation value of the output neuron @xmath70 .    @xmath0 an analogues algorithm for reconstruction of the energy in the case of cms detector has been applied .\\nthis scheme has shown significant distortion of spectrum shape , bad energy resolution and a big nonlinearity .\\nobviously , this is a consequence of the more complicated structure and bigger fluctuations of hadron showers in this case .\\n@xmath0a slightly different approach was used to determine the energy correction factors for the atlas detector @xcite using recurrent neural network with nearest neighbour feedback in the input layer and a single output giving the corrected energy value .\\nit was shown that the network performs satisfactory , but there is no comparison of the results obtained in this approach with results given by more conventional algorithms .\\n@xmath0the second possibility is to use the neural network for adjustment of the weights @xmath72 in ( 1 ) on event by event basis .\\nwe shall developed it in what follows .\\n@xmath0the idea of our method is : to tune the weights ( calibration coefficients ) @xmath47 in ( 1 ) for the four longitudinal readouts of the calorimetry system on event by event basis , using the available information about lateral and transversal development of the hadron shower .\\nsuch an approach is more flexible and allows to take into account the quite different behaviour ( for example  different e / h - ratio ) of the electromagnetic and hadron compartments of the cms calorimetry system .\\n@xmath0as it was mentioned above , the calorimeter system of cms is non - compensating . as a result\\n, there is difference between optimal calibration coefficients for jets , hadrons and electrons .\\ntherefore , in order to improve the energy reconstruction , it is preferable to identify the type of the initiator of the shower .\\n@xmath0 the data processing is divided in two steps : first , identification of the type of the incident particle and second , determination of the energy deposited in the calorimeter .\\ncorrespondingly , we use two - level neural network .\\nthe first level network classifies the hadron showers in four classes , respectively , initiated by :    * mainly electromagnetic interacting particles - @xmath73 , @xmath74 and @xmath75 .\\n* mainly strong interacting particles - hadrons .\\n* muons .\\n@xmath0the energy reconstruction is then performed by a dedicated network for each class of showers .\\nthe second level network has four subnets ( fig2.nn ) corresponding to the four longitudinal read - outs .\\nthe activation values of the four output neurons give the relevant correction factors for the weights in the sm .\\n@xmath0the neural network has @xmath76 input nodes :    * @xmath77 - the energy of the shower reconstructed using the standard method with @xmath53 for 300 gev * @xmath78 , @xmath79 , where @xmath80 is the energy deposited in the @xmath70th read - out . * @xmath81 inputs with signals from ecal * @xmath82 x @xmath83 inputs with signals from the three read - outs of hcal        @xmath0 in our reconstruction scheme , we take into account the energies deposited in a ( @xmath23 ) cone with transversal radius @xmath84 around the shower maximum .\\nthis corresponds to matrix of towers with size 41x41 for the ecal and with size 7x7 for the hcal . because the number of towers is too large , in order to reduce the number of network input neurons , we merge together transversal towers , by summing the energies in concentric squares as shown on fig . 3.nn .\\n@xmath0 during the training phase the nn stands in need of some additional neurons . the supplementary part of the network is represented on fig .\\noutputs ( out1 ... out4 ) of the basic network are connected with four hidden neurons with input function @xmath85 and activation function @xmath86 .\\nfour additional inputs ( extra - in1 ... extra - in4 ) multiply them with a value , proportional to the energy deposited in the corresponding read - out of the calorimeter .\\nthe output neuron ( out ) sums up the signals ( a(i)=i ) .\\nadditionally , we correct the output teaching samples with factor @xmath87 in order to obtain correct contribution from different energies to the error function ( 4 ) of the net . without this correction ,\\nthe contribution to the error function of events with low energies is minor and as a result , the connection weights of the nn are tuned to reconstruct well only high energy showers .\\nthe same correction is applied to the values of the extra inputs . during the learning phase\\n, the weights of the connections ( @xmath88 ) - @xmath89 , ( @xmath90)-@xmath91 and ( @xmath92 ) - @xmath93 , are treated like all other weights in the nn .\\nafter the learning is finalized , the supplementary part of the net is removed .\\n@xmath0the reconstructed energy is calculated as : @xmath94 where @xmath71 are activation values of the output neurons of nn , @xmath95 and @xmath80 are the energies deposited in the longitudinal read - outs .the coefficients @xmath96 are fixed during the learning of the nn .\\nvariations in the shower development are accounted by @xmath71 .\\n@xmath0for realization of the neural network we have used stuttgart neural network simulator ( snns ) @xcite . our experience has shown that this simulator is more flexible and efficient , than the widely used in the high energy physics neural network package jetnet @xcite .\\n@xmath0to test the performance of the neural network we have used the same sample of simulated events as in section 3 .\\nthe number of particles of each energy and type is 5000 , 3000 of them have been used to train the network and the rest 2000 for the test itself .\\nadditional samples of simulated events with energies different from those used during learning phase , have been used as well to carry out an independent test of the network .\\n@xmath0 the recognition of the type of the shower initiator , has been done by two different methods . in the first method ,\\nusing suitably chosen cuts , we separate consequently electrons from pions and jets and after that , pions from jets .\\nthe first cut we apply is the value of the shower pseudoradius in the electromagnetic calorimeter :    @xmath97    where @xmath98 is the energy deposited in the crystal @xmath99 with coordinates @xmath100,@xmath101 .\\nif @xmath102 has value in the interval ( 0.25,0.7 ) , the shower is considered as initiated by electron .\\nthe efficiency for recognition of electromagnetic showers is 99.95 @xmath7 .    @xmath0 the second step is separation of single hadron showers . for their recognition\\nseveral additional cuts have been used .\\nthe showers are classified as induced by single hadron if :    * @xmath103 * the energy deposited in the electromagnetic calorimeter corresponds to mip ( @xmath104 ) * @xmath105 , where @xmath106 .\\n* @xmath107 , @xmath108 , where @xmath109 and @xmath110 are the energies dissipated in the electromagnetic and hadron calorimeters correspondently .\\nthe energies are obtained using calibration coefficients for 300 gev pions .\\n@xmath0 if a shower does not obey any of the above conditions , it is classified as initiated by hadron jet .\\n@xmath0 the shower is classified as initiated by muon , if in all calorimeter compartments , the signal corresponds to the mip one .\\nthis simple requirement gives very high ( more @xmath111 ) efficiency of muon recognition .\\n@xmath0 the separation of electrons from jets and pions is efficient enough .\\nhowever , the efficiency for pion recognition varies with the energy between @xmath112 and @xmath113 .\\nthe situation with jets is even worse  the efficiency is between @xmath114 and @xmath115 ( see fig .\\n[ fig : nntef ] ) .\\n@xmath0 in order to get better efficiency of recognition we have used a neural network . as we can expect , the efficiency of recognition is much higher and even for pions and jets is more then @xmath116 ( fig .\\n[ fig : nntef ] ) .\\n@xmath0 of course it is possible to reach better efficiency of recognition using additional information from the central tracker , but for our purposes it is more important to classify the showers according to their specific energy distribution in the calorimetry system . then even in the case of wrong recognition of the initializing particle\\n, it is possible to reconstruct the energy of the shower with good accuracy . the reconstructed energy by nn for 300 gev jet showers , classified by nn as single hadron showers is shown on fig .  [ fig : nnjp ] .\\nthe reason for misidentification is that in those cases the shower characteristics are very close to the one for pion showers .\\nthis explains why is nevertheless the energy sufficiently well reconstructed .\\nthe same situation takes place for jets mixed with electrons .\\nthe part of wrongly classified events is small and the reconstruction of their energies is satisfactory .\\nthis can be seen on fig .\\n[ fig : nnwrong ] , where the reconstructed energy for 300 gev jets for all events is plotted and the contribution of the wrongly recognized events is shown .\\n@xmath0 the spectra of reconstructed energies for 300 gev jets by nn have clear gaussian shape ( see fig .\\n[ fig : nnjs ] ) and even at low energies there is no significant tails .\\nthe energy resolution obtained is    @xmath117    and is plotted on fig .\\n[ fig : nnh1r ] in comparison with the one obtained with the use of the h1 technique with weights fitted at every energy ( idealized case ) .\\nthe stochastic term is lower than this evaluated with the h1 technique ( @xmath118 ) mainly due to the improvement of the resolution for low energies .\\nthe constant term is slightly higher for nn - based solution .\\nlinearity is shown on fig .\\n[ fig : lin ] .\\nit can be improved further adding a small nn to the nn performing the energy reconstruction .\\n@xmath0a feed - forward neural network has been applied for reconstruction of the energy deposited in the cms detector by single hadrons and jets .\\nwe perform the reconstruction in two steps .\\nfirst , the showers are classified by the nn , according to the type of the initializing particle ( muon , electron , single hadron , hadron jet ) , a high enough efficiency of recognition has been achieved .\\nit is shown , that even if the shower is misidentified , its energy is reconstructed correctly . at the second step , for the every class of events ,\\na dedicated neural network evaluates the energy of the shower .\\na significant improvement of the energy resolution and linearity has been achieved in comparison with the ones acquired with the help of different weighting methods .\\nthe energy spectra have a gaussian shape and are free of tails .\\nwe would like to tank v. genchev and n. durmenov for numerous helpful discussions .      cms,technical proposal,cern / lhcc 94 - 38 cms - the electromagnetic calorimeter project , cern / lhcc 97 - 33,1997 .\\ncms - the hadron calorimeter project , cern / lhcc 97 - 31 , 1997 . v. abramov et al .\\n, `` studies of the response of the prototype cms hadron calorimeter , including magnetic field effects , to pion , electron and muon beam '' , cms note/2000 - 03 , cern,2000 .\\nw.braunschweig at al .\\n, nucl.instr . and meth .\\na265 , p. 419\\nw.braunschweig at al .\\n, nucl.instr . and meth .\\na275 , p. 246\\nf. ariztizabal at al .\\n, nucl.instr . and meth .\\na349 , p. 384\\na. astvatsaturov at al .\\n, dubna preprint , e13 - 94 - 522 , dubna , 1994 . m. kunze  application of neural networks in the analysis of multi - particle data  , world scientific ( 1994 )    b. denby , computer physics communication , 119 , p.219 , 1999 .\\ni.iashvili and a. kharchilava , \\n@xmath119 signal separation using a neural network .\\ns. banerjee , a. khan ,  application of neural networks in detecting higgs at cms \\ncms tn/96 - 023 t. maggipinto et al .\\n role of neural networks in the search of the higss boson at lhc  , bari - th/268 - 97 , may 1997 .\\nbusson , r. noberega , j. varela  modular neural networks for on - line event classification in high energy physics  , nuclear instruments and methods in physics research a410 ( 1998 ) 273 - 283 l. borissov , a .\\nkirkby , h. newman .\\ns. shevchenko \\nneural pion rejection in the cms pbwo@xmath120 crystal calorimeter using a neural network .\\nthe dependence of neutral pion rejection factor on crystal s off - pointing angle .\\n california institute of technology , pasadena , ca 91125 s.chattopadhyay , z. ahammed and y. p. viyogi  application of neural network for photon - hadron discrimination in a preshower detector in high energy heavy ion collisions  vecc calcutta-700064 ( india )    d.e .\\nrumelhart and j.l mcclelland , in the `` paralleled distributed processing '' , vol.1 . mit press , 1986 .\\ng. bortolotto et al . , \\nneural networks in experimental high - en ergy physics  , preprint 91/09/cb - 23 december 1991 m.riedmiller and h. braun , `` a direct adaptive method for faster backpropagation learning .\\nthe rprop algorithm '' , in proceedings of the ieee international conference on neural networks 1993 ( icnn 93 ) , 1993 .\\nc.peterson and e. hartman , `` explorations of the mean field theory learning algorithm '' neural networks 2 , 475 , 1989 .\\nr. borisyuk et .\\nal ,  gamma - ray energy determination using neural network algorithms for an imaging silicon calorimeter  , infn / ae-96/23 j. seixas  using neural networks to learn energy correction factors : a case study for the atlas calorimeter  ,\\ncern - th-98 - 191 snnsv4.0  user manual  1995 jetnet 3.0 - a versatile artificial neural network package cern - th .\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"abstract\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2418,\n        \"samples\": [\n          \" bayesian optimization has become a successful tool for hyperparameter optimization of machine learning algorithms , such as support vector machines or deep neural networks . despite its success , for large datasets , training and validating \\n a single configuration often takes hours , days , or even weeks , which limits the achievable performance . to accelerate hyperparameter optimization \\n , we propose a generative model for the validation error as a function of training set size , which is learned during the optimization process and allows exploration of preliminary configurations on small subsets , by extrapolating to the full dataset . \\n we construct a bayesian optimization procedure , dubbed , which models loss and training time as a function of dataset size and automatically trades off high information gain about the global optimum against computational cost . \\n experiments optimizing support vector machines and deep neural networks show that often finds high - quality solutions 10 to 100 times faster than other state - of - the - art bayesian optimization methods or the recently proposed bandit strategy hyperband . \",\n          \" the average length and average relaxation time of attractors in sequence processing neural networks are investigated . \\n the simulation results show that a critical point of @xmath0 , the loading ratio , is found . below the turning point , the average length is equal to the number of stored patterns ; conversely , the ratio of length and numbers of stored patterns , grow with an exponential dependence @xmath1 . moreover , we find that the logarithm of average relaxation time is only linearly associated with @xmath0 and the turning point of coupling degree is located for examining robustness of networks .    _ keywords : _ neural network ; asymmetric neural networks ; attractor ; relaxation time ; dilution factor    [ [ section ] ]    the dynamic behavior of hopfield model@xmath2 , a global symmetric coupling neural network , is relatively simple : the system relaxes to a fixed point corresponding to a stable patterns@xmath3 . \\n the latest result of its maximal stored capacity @xmath4 given by volk@xmath5 is @xmath6 because the synaptic connections in real biological neural networks have a highly degree of asymmetry and a real human idea consists of a set of patterns , many modification of hopfield model have been designed@xmath7 . in general \\n , an asymmetric connection strength may arise to recall a series of patterns or even chaos behavior , and can not be studied by conventional statistical methods because it disobeys detailed balance .    in this paper \\n , we study the behavior of attractors in a sequence processing model@xmath8 , a fully connected ising spin model , through numerical simulation with deterministic parallel dynamics . \\n moreover , the robustness is examined by locating critical coupling degrees since the loss of synaptic connections in the human brain may occur because of brain damage@xmath9 . \",\n          \" computational neuroscience models have been used for understanding neural dynamics in the brain and how they may be altered when physiological or other conditions change . \\n we review and develop a data - driven approach to neuroimaging data called the energy landscape analysis . \\n the methods are rooted in statistical physics theory , in particular , the ising model , also known as the ( pairwise ) maximum entropy model and boltzmann machine . \\n the methods have been applied to fitting electrophysiological data in neuroscience for a decade , but its use in neuroimaging data is still in its infancy . \\n we first review the methods and discuss some algorithms and technical aspects . \\n then , we apply the methods to functional magnetic resonance imaging data recorded from healthy individuals to inspect the relationship between the accuracy of fitting , the size of the brain system to be analyzed , and the data length . \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"section_names\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2107,\n        \"samples\": [\n          \"introduction\\ntheoretical background\\nresults and discussion\\nacknowledgements\",\n          \"introduction\\nanalytic theory of non-resonant wave-particle scattering\\ntest particle calculations and discussions\\nconclusions\",\n          \"introduction\\nrelativistic beta decay kinematics\\nnumerical simulations\\nthree neutrino case\\nsummary and discussion\\nacknowledgements\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["# Define Training and Validation Datasets\n","# --------------------------------------\n","print('Size of dataset: ', len(df))\n","\n","# --- index for validation dataset --- #\n","np.random.seed(35)\n","size = .1\n","val_n = round(len(df)*size)\n","\n","val_index = np.random.randint(0, len(df), val_n)\n","val_index\n","\n","# --- training dataset --- #\n","df_train = df.drop(val_index)\n","df_train = df_train['abstract'].to_list()\n","print('Size of training data: ', len(df_train))\n","print('Duplicate each training sentence: ')\n","df_train = [item for item in df_train for _ in range(10)]\n","print('Size of training data: ', len(df_train))\n","\n","\n","# --- validation dataset --- #\n","df_val = df.iloc[val_index]\n","df_val = df_val['abstract'].to_list()\n","print('Size of validation data: ', len(df_val))\n","print('Duplicate each validation sentence: ')\n","df_val = [item for item in df_val for _ in range(10)]\n","print('Size of training data: ', len(df_val))\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":142},"id":"QVseTqEL6tG_","executionInfo":{"status":"ok","timestamp":1712422180484,"user_tz":240,"elapsed":27,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"d4c7ac7a-7502-4769-a048-322a12bfa59e"},"execution_count":4,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Size of dataset:  2418\n","Size of training data:  2188\n","Duplicate each training sentence: \n","Size of training data:  21880\n","Size of validation data:  242\n","Duplicate each validation sentence: \n","Size of training data:  2420\n"]}]},{"cell_type":"code","source":["# Test examples\n","# ------------------\n","\n","test_text = \"\"\"\n","Neurons are the main components of nervous tissue in all animals\n","except sponges and Placozoa. Non-animals like plants and fungi do\n","not have nerve cells. Molecular evidence suggests that the ability to\n","generate electric signals first appeared in evolution some 700 to 800\n","million years ago, during the Tonian period. Predecessors of neurons\n","were the peptidergic secretory cells. They eventually gained new gene\n","modules which enabled cells to create post-synaptic scaffolds and ion\n","channels that generate fast electrical signals. The ability to generate\n","electric signals was a key innovation in the evolution of the nervous\n","system.\n","\"\"\"\n","\n","test_abstract = \"\"\"\n","Human frontocentral event-related potentials (FC-ERPs) are ubiquitous\n","neural correlates of cognition and control, but their generating\n","multiscale mechanisms remain mostly unknown. We used the Human\n","Neocortical Neurosolver(HNN)’s biophysical model of a canonical\n","neocortical circuit under exogenous thalamic and cortical drive to\n","simulate the cell and circuit mechanisms underpinning the P2, N2, and\n","P3 features of the FC-ERP observed after Stop-Signals in the\n","Stop-Signal task (SST). We demonstrate that a sequence of simulated\n","external thalamocortical and cortico-cortical drives can produce the\n","FC-ERP, similar to what has been shown for primary sensory cortices.\n","We used this model of the FC-ERP to examine likely circuit-mechanisms\n","underlying FC-ERP features that distinguish between successful and\n","failed action-stopping. We also tested their adherence to the\n","predictions of the horse-race model of the SST, with specific\n","hypotheses motivated by theoretical links between the P3 and Stop\n","process. These simulations revealed that a difference in P3 onset\n","between successful and failed Stops is most likely due to a later\n","arrival of thalamocortical drive in failed Stops, rather than, for\n","example, a difference in effective strength of the input. In contrast,\n","the same model predicted that early thalamocortical drives underpinning\n","the P2 and N2 differed in both strength and timing across stopping\n","accuracy conditions. Overall, this model generates novel testable\n","predictions of the thalamocortical dynamics underlying FC-ERP\n","generation during action-stopping. Moreover, it provides a detailed\n","cellular and circuit-level interpretation that supports links between\n","these macroscale signatures and predictions of the behavioral race\n","model. Significance statement The frontocentral event-related potential\n","(FC-ERP) is an easily-measurable neural correlate of cognition and\n","control. However, the cortical dynamics that produce this signature in\n","humans are complex, limiting the ability of researchers to make\n","predictions about its underlying mechanisms. In this study, we used the\n","biophysical model included in the open-source Human Neocortical\n","Neurosolver software to simulate and evaluate the likely cellular and\n","circuit mechanisms that underlie the FC-ERP in the Stop-Signal task. We\n","modeled mechanisms of the FC-ERP during successful and unsuccessful\n","stopping, generating testable predictions regarding Stop-associated\n","computations in human frontal cortex. Moreover, the resulting model\n","parameters provide a starting point for simulating mechanisms of the\n","FC-ERP and other frontal scalp EEG signatures in other task conditions\n","and contexts.\n","\"\"\""],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"Qy4M_zNsKF-T","executionInfo":{"status":"ok","timestamp":1712422180484,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"97585ce8-c74d-44ea-d49c-812fcd2f37c8"},"execution_count":5,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"markdown","source":["# MLM Training Pipeline"],"metadata":{"id":"DYphnxxKF1DW"}},{"cell_type":"code","source":["# ------------------------- #\n","# --- Training Pipeline --- #\n","# ------------------------- #\n","\n","from transformers import BartForSequenceClassification, BartTokenizer, AdamW, BartForConditionalGeneration\n","import torch\n","from torch import nn"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"UkaHlU3bF4wA","executionInfo":{"status":"ok","timestamp":1712422180676,"user_tz":240,"elapsed":201,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"1009e901-5e2c-4840-8f7b-719d90b52a96"},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["# model and tokenizer\n","# ------------------\n","\n","model_name = \"facebook/bart-base\"\n","\n","tokenizer = BartTokenizer.from_pretrained(\n","    model_name\n",")\n","\n","model = BartForConditionalGeneration.from_pretrained(\n","    model_name\n",")\n","\n","model.to(device)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["18a145b799a742718327d01f5a6095c6","158f9369c0fe41698af9487f5019131b","dcbe36b519c04b69a119b9c5552d5f89","7b292ed931a646b0bbedc0dfee07e4db","8887f1e57e1a4492a68937185a15e4b6","19ce3e5dccff43c28e4c1bc48c76f4a3","672227b42afc4af2b28e0cea9b49b0a5","d56003e2ecc541b8a31dee066ceaedb7","833ecd615e2c43ce84473138482336e6","ac893f761ace4e9ba2b84ffc628cbd42","838065164e7f4aac89e3aab0bba4129a","ed9331de07c64c489e52bf1c6ad4f3aa","9195c31428a0400ea064f20510ad3c75","1d5b158f1caf48a3a76542112119af5d","81f9a9cc3ef9424ab3b8de09f406252b","eacd577a91ab44db97070e6dc5582070","e9d74b7de3fb40ef9d04268594ed761c","e5b2962bfe7542f7bf8e321f0cc19896","2179f15774ec481d8689f73084fe22e9","aa43a3a13f4942af9c4b4e2843c80597","021a611cda154097a7a1abdcbea354f2","1f15fd15fecf4c55896e44a8f2118b25","ce64ceecad8346bea68870f62ad5bc43","9c89b3b9b4754132a3beb6c9585ea415","f3ceaa229f854b4aa12c8a3de67bbc64","68ae4448f561485e815aa0742709bffc","841e1bd4a48a4c0db54598f36b560794","95b8fba9d5c3470baba50d8452c1181a","f207de4171724d9098c40f4290df06b9","8e5f27409e144249af7d32b73f520e75","2d3a7e80f9294b11a985afd39bb3187a","a6e965839d3b4240830612c340add746","1dcd5e0210af4b90863586178ae80f1b","960e28e1db0d4eba85c020370e5c260b","4f97525ece9a4743b0c9a6834d49ce1a","c86a1587136248348188069d51b6d212","8283c30986e44cf4a921b73bd43bc5ec","ad31db176dc64b518a2599404ce0573b","8a0e2c61ce58473c9de54270a9717e73","cb0468aac45849cfaf4378c1db26114d","c5f5ba16b4b74221bb1924afc60b7517","ebda7f4d336541f7a651ebdfa7855c04","487864fbf08049b8ad4c725be03537a9","ddb998fa9cab4671b55c9473579e5ac3","6f24c4de0bf24248b6d333a2b14447ce","45b78cdf80464175993f818c67d0fd98","19f715289d6b4c66b4432e398e3be9ed","fa137ae3569d499f8b47112e25a88833","e47cf5e886f7442586e50da66081ffa5","7d040032a4db4c42836e27cc9bef20e7","b12fa8b54d9148a599373f2650b2e047","dd9ea816f7ad405780b6e65d3896cdc3","8c791a6b35944dea9318188782731a51","0b1e85f75991462f81b71f7fed0b0d81","a10f5bd2c13f452dbf7c0c07d5d6d9e1"]},"id":"NhF4kSpLGkrl","executionInfo":{"status":"ok","timestamp":1712422190467,"user_tz":240,"elapsed":9800,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"71cf5f4a-98f1-4e28-e28a-a551de95fbf4"},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n","The secret `HF_TOKEN` does not exist in your Colab secrets.\n","To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n","You will be able to reuse this secret in all of your notebooks.\n","Please note that authentication is recommended but still optional to access public models or datasets.\n","  warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"18a145b799a742718327d01f5a6095c6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ed9331de07c64c489e52bf1c6ad4f3aa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ce64ceecad8346bea68870f62ad5bc43"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["config.json:   0%|          | 0.00/1.72k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"960e28e1db0d4eba85c020370e5c260b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["model.safetensors:   0%|          | 0.00/558M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6f24c4de0bf24248b6d333a2b14447ce"}},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["BartForConditionalGeneration(\n","  (model): BartModel(\n","    (shared): Embedding(50265, 768, padding_idx=1)\n","    (encoder): BartEncoder(\n","      (embed_tokens): Embedding(50265, 768, padding_idx=1)\n","      (embed_positions): BartLearnedPositionalEmbedding(1026, 768)\n","      (layers): ModuleList(\n","        (0-5): 6 x BartEncoderLayer(\n","          (self_attn): BartSdpaAttention(\n","            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","          (activation_fn): GELUActivation()\n","          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n","          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n","          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","        )\n","      )\n","      (layernorm_embedding): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","    )\n","    (decoder): BartDecoder(\n","      (embed_tokens): Embedding(50265, 768, padding_idx=1)\n","      (embed_positions): BartLearnedPositionalEmbedding(1026, 768)\n","      (layers): ModuleList(\n","        (0-5): 6 x BartDecoderLayer(\n","          (self_attn): BartSdpaAttention(\n","            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (activation_fn): GELUActivation()\n","          (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","          (encoder_attn): BartSdpaAttention(\n","            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n","            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (encoder_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n","          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n","          (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","        )\n","      )\n","      (layernorm_embedding): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n","    )\n","  )\n","  (lm_head): Linear(in_features=768, out_features=50265, bias=False)\n",")"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["run_test = False"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"qDO5m2jMMmMx","executionInfo":{"status":"ok","timestamp":1712422190467,"user_tz":240,"elapsed":30,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"eeca88ca-7c7a-491e-b43f-5ee71011ccc4"},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["if run_test:\n","    try:\n","        test_tokenized_data\n","    except:\n","        test_tokenized_data = False\n","\n","    if not test_tokenized_data:\n","        test_tokenized_data = tokenizer(\n","            df_train,\n","            max_length=512,\n","            truncation=True,\n","            padding='max_length',\n","            return_tensors=\"pt\",\n","        )"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"btRx-ZBfLIuE","executionInfo":{"status":"ok","timestamp":1712422190467,"user_tz":240,"elapsed":13,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"090a24ab-de09-4431-cbfd-cd116472ca86"},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["# define Datast class\n","# ------------------\n","\n","class ContinuedPretrainData(Dataset):\n","    def __init__(self, base_data, tokenizer, device, max_len=512):\n","        self.max_len = max_len\n","        self.tokenizer = tokenizer\n","        self.data = []\n","        self.labels = []\n","        self.attention_mask = []\n","\n","        tokenized_examples = tokenizer(\n","            df_train,\n","            max_length=512,\n","            truncation=True,\n","            padding='max_length',\n","            return_tensors=\"pt\"\n","        )\n","\n","        input_ids = tokenized_examples['input_ids']\n","\n","        # mask tokens\n","        mask_indices = torch.rand(input_ids.shape) < 0.15  # 15% probability\n","\n","        # un-mask padding tokens\n","        mask_indices[tokenized_examples['attention_mask'] == 0] = False\n","        mask_indices[:, 0] = False # un-mask first token in sequence\n","        # mask_indices[input_ids == 0] = False  # un-mask 0 (bos) tokens\n","        # mask_indices[input_ids == 1] = False  # un-mask 1 (pad) tokens\n","        mask_indices[input_ids == 2] = False  # un-mask 2 (eos) tokens\n","\n","        masked_tokens = input_ids.clone()\n","        masked_tokens[mask_indices] = tokenizer.mask_token_id\n","        masked_tokens = torch.tensor(masked_tokens)\n","\n","        # Generate labels from masked tokens\n","        labels = input_ids.clone()\n","        labels[~mask_indices] = -100\n","        labels[mask_indices] = input_ids[mask_indices]\n","        labels = torch.tensor(labels)\n","\n","        # send data to device\n","        self.data = masked_tokens.to(device)\n","        self.labels = labels.to(device)\n","        self.attention_mask = tokenized_examples['attention_mask'].to(device)\n","\n","    def __len__(self):\n","        return self.data.shape[0]\n","\n","    def __getitem__(self, index):\n","        return {'inputs': self.data[index],\n","                'labels': self.labels[index],\n","                'attention_mask': self.attention_mask[index],\n","        }"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"Bst3yrxCGSML","executionInfo":{"status":"ok","timestamp":1712422190467,"user_tz":240,"elapsed":12,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"a21e6937-0750-429f-df0a-51c726b56147"},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["%%capture\n","\n","# initialize datasets\n","train_dataset = ContinuedPretrainData(\n","    df_train,\n","    tokenizer,\n","    device\n",")\n","\n","val_dataset = ContinuedPretrainData(\n","    df_val,\n","    tokenizer,\n","    device\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"tp_BmSVjGUXD","executionInfo":{"status":"ok","timestamp":1712422496373,"user_tz":240,"elapsed":305917,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"19b23e32-3661-4648-ac82-01f3b0858207"},"execution_count":11,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["batch_size = 4\n","\n","# initialize dataloaders\n","train_dataloader = DataLoader(\n","    train_dataset,\n","    batch_size=batch_size,\n","    shuffle=True\n",")\n","\n","val_dataloader = DataLoader(\n","    val_dataset,\n","    batch_size=batch_size,\n","    shuffle=False\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"-mEs1AUyd_V4","executionInfo":{"status":"ok","timestamp":1712422496374,"user_tz":240,"elapsed":35,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"50d66fbc-b009-478e-ba03-784a9f01a4e2"},"execution_count":12,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["print(\n","    'Masked token ID:\\t',\n","    tokenizer.mask_token_id,\n","    '\\n'\n","    'Length of Dataset:\\t',\n","    len(train_dataset),\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":53},"id":"Ba91xEs7WRwv","executionInfo":{"status":"ok","timestamp":1712422496374,"user_tz":240,"elapsed":18,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"b91326be-d13f-4a99-a5da-6570d5813779"},"execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Masked token ID:\t 50264 \n","Length of Dataset:\t 21880\n"]}]},{"cell_type":"code","source":["print(\n","    train_dataset[0]['inputs'][0:5],\n","    '\\n',\n","    train_dataset[0]['labels'][0:5],\n","    sep='',\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":53},"id":"IOLSgJ4PKLlC","executionInfo":{"status":"ok","timestamp":1712422496374,"user_tz":240,"elapsed":16,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"5916137d-12e7-4553-bc21-50ff4d8a02da"},"execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["tensor([    0, 47621,  3783,    16,  1687], device='cuda:0')\n","tensor([-100, -100, -100, -100, -100], device='cuda:0')\n"]}]},{"cell_type":"code","source":["# next(iter(train_dataloader))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"BkU40-2vHOnh","executionInfo":{"status":"ok","timestamp":1712422496374,"user_tz":240,"elapsed":15,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"6556c23a-6bd0-4a1f-e56f-ffa5eb274cb8"},"execution_count":15,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["optimizer = torch.optim.AdamW(\n","    model.parameters(),\n","    lr=0.00001\n",")\n","\n","reinitialize_base_model = False\n","\n","if reinitialize_base_model:\n","    model = BartForConditionalGeneration.from_pretrained(\n","        \"facebook/bart-base\"\n","    )\n","    model.to(device)\n","\n","def continued_train_loop(\n","        train_dataloader,\n","        val_dataloader,\n","        model,\n","        optimizer,\n","        reporting_interval=50,\n","        save_state=False,\n","        out_directory='states/',\n","        out_fname='mlm_model_state',\n","    ) -> dict:\n","    \"\"\"\n","    Complete a single pass through the training dataloader\n","    \"\"\"\n","\n","    # --- Training Loop --- #\n","    model.train()  # training mode\n","    train_epoch_loss = 0\n","    val_epoch_loss = 0\n","    train_acc_history = []  # store training accuracies for each batch\n","    train_loss_history = []  # store losses for each batch\n","    val_acc_history = {'correct':0, 'total':0}  # store validation accuracies for each batch\n","\n","    for batch, example in enumerate(train_dataloader):\n","\n","        inputs = {\n","            \"input_ids\": example[\"inputs\"],\n","            \"labels\": example[\"labels\"],\n","            \"attention_mask\": example[\"attention_mask\"],\n","        }\n","\n","        # forward pass\n","        outputs = model(**inputs)\n","\n","        # get loss\n","        loss = outputs.loss\n","\n","        # backward pass\n","        loss.backward()\n","        optimizer.step()\n","        optimizer.zero_grad()\n","\n","        train_epoch_loss += loss.item()\n","\n","        # calculate accuracy on masked tokens\n","        mask = (inputs[\"input_ids\"] == tokenizer.mask_token_id)  # identify masked tokens\n","        masked_logits = outputs.logits[mask]\n","        predictions = torch.argmax(masked_logits, dim=-1)  # get predicted token index\n","        flat_labels = inputs[\"labels\"][mask]  # filter labels on masked tokens\n","        correct = (predictions == flat_labels).sum().item()\n","        train_acc = correct / len(predictions)  # calculate accuracy\n","        train_acc = round(train_acc, 5)\n","\n","        # training progress\n","        if int(batch + 1) % reporting_interval == 0:\n","            print(f'\\tFinished batches: {str(batch + 1)}')\n","            print(f'\\tTrain Loss: {round(loss.item(), 4)}')\n","            print(f'\\tTrain Acc: {train_acc}')\n","\n","        train_acc_history.append(train_acc)\n","        train_loss_history.append(loss.item())\n","\n","    # Print average training loss\n","    avg_train_loss = round(train_epoch_loss / len(train_loss_history), 5)\n","    print(f\"Average training loss: {avg_train_loss}\")\n","\n","    # --- Validation Loop --- #\n","    model.eval()  # evaluation mode\n","    with torch.no_grad():  # disable gradient calculation\n","\n","        for batch, example in enumerate(val_dataloader):\n","            inputs = {\n","                \"input_ids\": example[\"inputs\"],\n","                \"labels\": example[\"labels\"],\n","                \"attention_mask\": example[\"attention_mask\"],\n","            }\n","            outputs = model(**inputs)\n","            loss = outputs.loss\n","\n","            val_epoch_loss += loss.item()\n","\n","            # calculate accuracy\n","            mask = (inputs[\"input_ids\"] == tokenizer.mask_token_id)\n","            masked_logits = outputs.logits[mask]\n","            predictions = torch.argmax(masked_logits, dim=-1)\n","            flat_labels = inputs[\"labels\"][mask]\n","            correct = (predictions == flat_labels).sum().item()\n","            # val_acc = correct / len(predictions)\n","            # val_acc = round(val_acc, 5)\n","\n","            val_acc_history['correct']+=correct\n","            val_acc_history['total']+=len(predictions)\n","\n","    # Print training and validation results\n","    # avg_val_loss = round(val_epoch_loss / len(val_dataloader), 5)\n","    # print(f\"Average validation loss: {avg_val_loss}\")\n","    avg_val_loss = round(val_epoch_loss / batch, 5)\n","    print(f\"Average validation loss: {avg_val_loss}\")\n","    overall_val_acc = round(val_acc_history['correct'] / val_acc_history['total'], 5)\n","\n","\n","    # Print final accuracies\n","    print(f\"Final Train Accuracy: {round(train_acc, 5)}\")\n","    print(f\"Validation Accuracy: {overall_val_acc}\")\n","\n","    if save_state:\n","        # torch.save(model.state_dict(), 'states/model_state.pth')\n","        model.save_pretrained(out_directory+out_fname)\n","\n","    # Update return dictionary\n","    return {\n","        'train_loss_history': train_loss_history,\n","        'train_acc_history': train_acc_history,\n","        'val_acc': overall_val_acc,\n","        'val_loss': avg_val_loss,\n","    }\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"pfMdpkoq2JxX","executionInfo":{"status":"ok","timestamp":1712422497478,"user_tz":240,"elapsed":1118,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"6fff9090-5d73-4191-fabc-29eb994c60f5"},"execution_count":16,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["run_single_epoch = False\n","\n","if run_single_epoch:\n","    model_results = continued_train_loop(\n","            train_dataloader,\n","            val_dataloader,\n","            model,\n","            optimizer,\n","            save_state=True\n","        )"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"0b1jRDCs4pl6","executionInfo":{"status":"ok","timestamp":1712427144293,"user_tz":240,"elapsed":4581081,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"681a8cb1-06c2-45c8-9865-c0a45a835810"},"execution_count":19,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["\tFinished batches: 50\n","\tTrain Loss: 6.9703\n","\tTrain Acc: 0.06818\n","\tFinished batches: 100\n","\tTrain Loss: 5.7556\n","\tTrain Acc: 0.14815\n","\tFinished batches: 150\n","\tTrain Loss: 6.7847\n","\tTrain Acc: 0.0625\n","\tFinished batches: 200\n","\tTrain Loss: 6.2932\n","\tTrain Acc: 0.11735\n","\tFinished batches: 250\n","\tTrain Loss: 6.1768\n","\tTrain Acc: 0.11554\n","\tFinished batches: 300\n","\tTrain Loss: 5.7844\n","\tTrain Acc: 0.11976\n","\tFinished batches: 350\n","\tTrain Loss: 5.5548\n","\tTrain Acc: 0.14583\n","\tFinished batches: 400\n","\tTrain Loss: 5.7066\n","\tTrain Acc: 0.13171\n","\tFinished batches: 450\n","\tTrain Loss: 5.5744\n","\tTrain Acc: 0.20979\n","\tFinished batches: 500\n","\tTrain Loss: 5.5584\n","\tTrain Acc: 0.14706\n","\tFinished batches: 550\n","\tTrain Loss: 5.2021\n","\tTrain Acc: 0.22\n","\tFinished batches: 600\n","\tTrain Loss: 5.3196\n","\tTrain Acc: 0.24031\n","\tFinished batches: 650\n","\tTrain Loss: 4.9804\n","\tTrain Acc: 0.26056\n","\tFinished batches: 700\n","\tTrain Loss: 4.9782\n","\tTrain Acc: 0.22907\n","\tFinished batches: 750\n","\tTrain Loss: 5.0407\n","\tTrain Acc: 0.24521\n","\tFinished batches: 800\n","\tTrain Loss: 5.0173\n","\tTrain Acc: 0.19091\n","\tFinished batches: 850\n","\tTrain Loss: 5.0612\n","\tTrain Acc: 0.24468\n","\tFinished batches: 900\n","\tTrain Loss: 4.6564\n","\tTrain Acc: 0.26154\n","\tFinished batches: 950\n","\tTrain Loss: 4.7497\n","\tTrain Acc: 0.25969\n","\tFinished batches: 1000\n","\tTrain Loss: 4.1573\n","\tTrain Acc: 0.31188\n","\tFinished batches: 1050\n","\tTrain Loss: 4.3191\n","\tTrain Acc: 0.29146\n","\tFinished batches: 1100\n","\tTrain Loss: 4.517\n","\tTrain Acc: 0.26923\n","\tFinished batches: 1150\n","\tTrain Loss: 4.3504\n","\tTrain Acc: 0.25397\n","\tFinished batches: 1200\n","\tTrain Loss: 4.4221\n","\tTrain Acc: 0.31553\n","\tFinished batches: 1250\n","\tTrain Loss: 4.3618\n","\tTrain Acc: 0.27751\n","\tFinished batches: 1300\n","\tTrain Loss: 3.9955\n","\tTrain Acc: 0.31837\n","\tFinished batches: 1350\n","\tTrain Loss: 4.3001\n","\tTrain Acc: 0.27184\n","\tFinished batches: 1400\n","\tTrain Loss: 3.8288\n","\tTrain Acc: 0.34545\n","\tFinished batches: 1450\n","\tTrain Loss: 3.8908\n","\tTrain Acc: 0.33745\n","\tFinished batches: 1500\n","\tTrain Loss: 3.9311\n","\tTrain Acc: 0.36864\n","\tFinished batches: 1550\n","\tTrain Loss: 3.4306\n","\tTrain Acc: 0.39024\n","\tFinished batches: 1600\n","\tTrain Loss: 3.0595\n","\tTrain Acc: 0.44068\n","\tFinished batches: 1650\n","\tTrain Loss: 3.8634\n","\tTrain Acc: 0.34545\n","\tFinished batches: 1700\n","\tTrain Loss: 3.3846\n","\tTrain Acc: 0.40465\n","\tFinished batches: 1750\n","\tTrain Loss: 2.8726\n","\tTrain Acc: 0.56757\n","\tFinished batches: 1800\n","\tTrain Loss: 4.2841\n","\tTrain Acc: 0.33663\n","\tFinished batches: 1850\n","\tTrain Loss: 4.1728\n","\tTrain Acc: 0.3046\n","\tFinished batches: 1900\n","\tTrain Loss: 3.5872\n","\tTrain Acc: 0.42069\n","\tFinished batches: 1950\n","\tTrain Loss: 3.4949\n","\tTrain Acc: 0.44172\n","\tFinished batches: 2000\n","\tTrain Loss: 4.2741\n","\tTrain Acc: 0.30319\n","\tFinished batches: 2050\n","\tTrain Loss: 3.6893\n","\tTrain Acc: 0.39375\n","\tFinished batches: 2100\n","\tTrain Loss: 3.0924\n","\tTrain Acc: 0.44762\n","\tFinished batches: 2150\n","\tTrain Loss: 3.7101\n","\tTrain Acc: 0.3427\n","\tFinished batches: 2200\n","\tTrain Loss: 3.3883\n","\tTrain Acc: 0.40226\n","\tFinished batches: 2250\n","\tTrain Loss: 3.581\n","\tTrain Acc: 0.3956\n","\tFinished batches: 2300\n","\tTrain Loss: 3.5224\n","\tTrain Acc: 0.38994\n","\tFinished batches: 2350\n","\tTrain Loss: 3.204\n","\tTrain Acc: 0.45614\n","\tFinished batches: 2400\n","\tTrain Loss: 3.3399\n","\tTrain Acc: 0.43602\n","\tFinished batches: 2450\n","\tTrain Loss: 2.497\n","\tTrain Acc: 0.5614\n","\tFinished batches: 2500\n","\tTrain Loss: 3.4928\n","\tTrain Acc: 0.44091\n","\tFinished batches: 2550\n","\tTrain Loss: 3.2675\n","\tTrain Acc: 0.4375\n","\tFinished batches: 2600\n","\tTrain Loss: 3.2666\n","\tTrain Acc: 0.48905\n","\tFinished batches: 2650\n","\tTrain Loss: 3.0426\n","\tTrain Acc: 0.4322\n","\tFinished batches: 2700\n","\tTrain Loss: 3.3226\n","\tTrain Acc: 0.43541\n","\tFinished batches: 2750\n","\tTrain Loss: 3.047\n","\tTrain Acc: 0.45238\n","\tFinished batches: 2800\n","\tTrain Loss: 3.1618\n","\tTrain Acc: 0.42188\n","\tFinished batches: 2850\n","\tTrain Loss: 3.6087\n","\tTrain Acc: 0.41825\n","\tFinished batches: 2900\n","\tTrain Loss: 4.2491\n","\tTrain Acc: 0.29545\n","\tFinished batches: 2950\n","\tTrain Loss: 3.3764\n","\tTrain Acc: 0.45509\n","\tFinished batches: 3000\n","\tTrain Loss: 3.8474\n","\tTrain Acc: 0.32584\n","\tFinished batches: 3050\n","\tTrain Loss: 2.7005\n","\tTrain Acc: 0.45324\n","\tFinished batches: 3100\n","\tTrain Loss: 3.2498\n","\tTrain Acc: 0.46569\n","\tFinished batches: 3150\n","\tTrain Loss: 3.8573\n","\tTrain Acc: 0.40086\n","\tFinished batches: 3200\n","\tTrain Loss: 3.2974\n","\tTrain Acc: 0.46405\n","\tFinished batches: 3250\n","\tTrain Loss: 3.2058\n","\tTrain Acc: 0.47317\n","\tFinished batches: 3300\n","\tTrain Loss: 3.6164\n","\tTrain Acc: 0.42754\n","\tFinished batches: 3350\n","\tTrain Loss: 3.8866\n","\tTrain Acc: 0.36158\n","\tFinished batches: 3400\n","\tTrain Loss: 3.1069\n","\tTrain Acc: 0.44493\n","\tFinished batches: 3450\n","\tTrain Loss: 2.9832\n","\tTrain Acc: 0.5122\n","\tFinished batches: 3500\n","\tTrain Loss: 2.8005\n","\tTrain Acc: 0.50296\n","\tFinished batches: 3550\n","\tTrain Loss: 3.645\n","\tTrain Acc: 0.39496\n","\tFinished batches: 3600\n","\tTrain Loss: 3.3639\n","\tTrain Acc: 0.43972\n","\tFinished batches: 3650\n","\tTrain Loss: 3.2073\n","\tTrain Acc: 0.39706\n","\tFinished batches: 3700\n","\tTrain Loss: 2.6791\n","\tTrain Acc: 0.54074\n","\tFinished batches: 3750\n","\tTrain Loss: 2.6946\n","\tTrain Acc: 0.55629\n","\tFinished batches: 3800\n","\tTrain Loss: 3.5437\n","\tTrain Acc: 0.38\n","\tFinished batches: 3850\n","\tTrain Loss: 2.7668\n","\tTrain Acc: 0.5298\n","\tFinished batches: 3900\n","\tTrain Loss: 3.3172\n","\tTrain Acc: 0.45399\n","\tFinished batches: 3950\n","\tTrain Loss: 3.3912\n","\tTrain Acc: 0.45638\n","\tFinished batches: 4000\n","\tTrain Loss: 2.364\n","\tTrain Acc: 0.58228\n","\tFinished batches: 4050\n","\tTrain Loss: 3.5739\n","\tTrain Acc: 0.36036\n","\tFinished batches: 4100\n","\tTrain Loss: 2.9746\n","\tTrain Acc: 0.42051\n","\tFinished batches: 4150\n","\tTrain Loss: 3.239\n","\tTrain Acc: 0.4183\n","\tFinished batches: 4200\n","\tTrain Loss: 3.2772\n","\tTrain Acc: 0.41961\n","\tFinished batches: 4250\n","\tTrain Loss: 3.1664\n","\tTrain Acc: 0.46575\n","\tFinished batches: 4300\n","\tTrain Loss: 3.0401\n","\tTrain Acc: 0.4739\n","\tFinished batches: 4350\n","\tTrain Loss: 2.8385\n","\tTrain Acc: 0.49045\n","\tFinished batches: 4400\n","\tTrain Loss: 2.6634\n","\tTrain Acc: 0.5288\n","\tFinished batches: 4450\n","\tTrain Loss: 2.5596\n","\tTrain Acc: 0.50549\n","\tFinished batches: 4500\n","\tTrain Loss: 3.6614\n","\tTrain Acc: 0.36735\n","\tFinished batches: 4550\n","\tTrain Loss: 2.8997\n","\tTrain Acc: 0.54936\n","\tFinished batches: 4600\n","\tTrain Loss: 2.9892\n","\tTrain Acc: 0.46531\n","\tFinished batches: 4650\n","\tTrain Loss: 2.6994\n","\tTrain Acc: 0.51822\n","\tFinished batches: 4700\n","\tTrain Loss: 2.9605\n","\tTrain Acc: 0.5\n","\tFinished batches: 4750\n","\tTrain Loss: 3.1744\n","\tTrain Acc: 0.45455\n","\tFinished batches: 4800\n","\tTrain Loss: 2.387\n","\tTrain Acc: 0.55085\n","\tFinished batches: 4850\n","\tTrain Loss: 2.8001\n","\tTrain Acc: 0.49339\n","\tFinished batches: 4900\n","\tTrain Loss: 2.7277\n","\tTrain Acc: 0.46067\n","\tFinished batches: 4950\n","\tTrain Loss: 2.9375\n","\tTrain Acc: 0.47107\n","\tFinished batches: 5000\n","\tTrain Loss: 2.7329\n","\tTrain Acc: 0.52239\n","\tFinished batches: 5050\n","\tTrain Loss: 3.0972\n","\tTrain Acc: 0.44022\n","\tFinished batches: 5100\n","\tTrain Loss: 2.685\n","\tTrain Acc: 0.49785\n","\tFinished batches: 5150\n","\tTrain Loss: 3.0773\n","\tTrain Acc: 0.48\n","\tFinished batches: 5200\n","\tTrain Loss: 2.6856\n","\tTrain Acc: 0.54183\n","\tFinished batches: 5250\n","\tTrain Loss: 3.2849\n","\tTrain Acc: 0.45492\n","\tFinished batches: 5300\n","\tTrain Loss: 2.404\n","\tTrain Acc: 0.56667\n","\tFinished batches: 5350\n","\tTrain Loss: 2.5466\n","\tTrain Acc: 0.51942\n","\tFinished batches: 5400\n","\tTrain Loss: 2.8105\n","\tTrain Acc: 0.50323\n","\tFinished batches: 5450\n","\tTrain Loss: 2.6642\n","\tTrain Acc: 0.49438\n","Average training loss: 3.73959\n"]},{"output_type":"stream","name":"stderr","text":["Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n","Non-default generation parameters: {'early_stopping': True, 'num_beams': 4, 'no_repeat_ngram_size': 3, 'forced_bos_token_id': 0, 'forced_eos_token_id': 2}\n"]},{"output_type":"stream","name":"stdout","text":["Average validation loss: 2.36896\n","Average validation loss: 2.3694\n","Final Train Accuracy: 0.48598\n","Validation Accuracy: 0.56384\n"]}]},{"cell_type":"code","source":["assert False"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":144},"id":"IH1K7iWz5a_w","executionInfo":{"status":"error","timestamp":1712422497478,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"bfed9249-d3e4-438d-e142-8c84ecc0b9b9"},"execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}},{"output_type":"error","ename":"AssertionError","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)","\u001b[0;32m<ipython-input-18-a871fdc9ebee>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mAssertionError\u001b[0m: "]}]},{"cell_type":"markdown","source":["# Training Loop"],"metadata":{"id":"kACeE5e3nS7g"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"n8JnI_HjPEzc"}},{"cell_type":"code","source":["# --- Load Previous Epoch Results --- #\n","all_epoch_results = pd.read_pickle(\n","    'states/mlm_state_checkpoints/all_epoch_results.pkl'\n",")\n","\n","all_epoch_results"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"3NIV8tazoAu_","executionInfo":{"status":"ok","timestamp":1712427145991,"user_tz":240,"elapsed":1318,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}},"outputId":"d7d3d139-bdd0-43d4-9048-0fd243a632c7"},"execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "]},"metadata":{}}]},{"cell_type":"code","source":["# --- Setup for Training Loop --- #\n","\n","epoch_results = {}\n","start_epoch = 0\n","num_epochs = 10\n","\n","save_all = True\n","load_checkpoint = True\n","\n","# --- Filepaths --- #\n","\n","checkpoint_path = 'states/mlm_state_checkpoints/'\n","\n","if save_all:\n","    out_folder = 'states/mlm_state_checkpoints/'\n","else:\n","    out_folder = 'states/'\n","\n","# --- Get Last Checkpoint Model --- #\n","folder_names = [\n","    name for name in os.listdir('states/mlm_state_checkpoints/')\n","    if os.path.isdir(os.path.join('states/mlm_state_checkpoints/', name))\n","]\n","previous_runs = [\n","    int(file.split('_')[0]) for file in folder_names if file.split('_')[0].isdigit()\n","]\n","\n","if load_checkpoint:\n","    checkpoint_fname = [\n","        name for name in folder_names\n","        if name.split('_')[0] == str(max(previous_runs))\n","    ]\n","    checkpoint_fname = checkpoint_fname[0]\n","\n","    print(f'Loading checkpoint from folder: {checkpoint_fname}')\n","\n","    model = BartForConditionalGeneration.from_pretrained(\n","        checkpoint_path+checkpoint_fname,\n","    )\n","    model.to(device)\n","    start_epoch = max(previous_runs)\n","\n","    print(f'Next loop will start at epoch {start_epoch+1}')\n"],"metadata":{"id":"V6dDMhTULVj6","executionInfo":{"status":"aborted","timestamp":1712422497479,"user_tz":240,"elapsed":337612,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Run Training Loop"],"metadata":{"id":"crv6BetQPHhx"}},{"cell_type":"code","source":["for epoch in range(start_epoch, num_epochs):\n","    run = epoch + 1\n","    print(f'Running epoch {run}')\n","    print('Time:',time.strftime(\"%H:%M\"))\n","\n","    if save_all:\n","        save_run = True\n","    else:\n","        if run == num_epochs:\n","            save_run = True\n","        else:\n","            save_run = False\n","\n","    out_name = f'{run}_model_state'\n","\n","    epoch_results[run] = continued_train_loop(\n","        train_dataloader,\n","        val_dataloader,\n","        model,\n","        optimizer,\n","        save_state=save_run,\n","        out_fname=out_name,\n","        out_directory=out_folder,\n","    )\n","\n","    interim_epoch_results = pd.DataFrame(epoch_results).T\n","    interim_epoch_results.to_pickle(\n","        out_folder+'interim_epoch_results.pkl'\n","    )\n","\n","print('Time:',time.strftime(\"%H:%M\"))"],"metadata":{"id":"6lj7Vgn61VRA","executionInfo":{"status":"aborted","timestamp":1712422497761,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Save epoch Results"],"metadata":{"id":"wkjdHCYgrWaD"}},{"cell_type":"code","source":["assert False"],"metadata":{"id":"IPS6iu46r0bs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["rewrite_results = False\n","\n","# average values for histories for training acc / training loss\n","def avg_list(row):\n","    return sum(row)/len(row)\n","\n","interim_epoch_results['avg_train_acc'] = interim_epoch_results['train_acc_history'].apply(avg_list)\n","interim_epoch_results['avg_train_loss'] = interim_epoch_results['train_loss_history'].apply(avg_list)\n","\n","if rewrite_results:\n","    interim_epoch_results.to_pickle('states/mlm_state_checkpoints/epoch_results.pkl')\n","else:\n","    all_epoch_results = pd.read_pickle('states/mlm_state_checkpoints/all_epoch_results.pkl')\n","    all_epoch_results = pd.concat([all_epoch_results, interim_epoch_results])\n","    all_epoch_results.to_pickle('states/mlm_state_checkpoints/all_epoch_results.pkl')\n","\n","all_epoch_results[\n","    ['val_acc', 'val_loss', 'avg_train_acc', 'avg_train_loss']\n","]"],"metadata":{"id":"p1bWCUVvvWMD","executionInfo":{"status":"aborted","timestamp":1712422497761,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"_UnBfwOwwIEv","executionInfo":{"status":"aborted","timestamp":1712422497761,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# End"],"metadata":{"id":"4QaXjVbdnCXH"}},{"cell_type":"markdown","source":["# Troubleshooting"],"metadata":{"id":"dDbYvKM8plEg"}},{"cell_type":"code","source":["assert False"],"metadata":{"id":"U7QgH6tGNEDS","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# # --- old method --- #\n","\n","# # filter out short examples\n","# test_input_ids = test_tokenized_data['input_ids'][test_tokenized_data['attention_mask'][:, 512 - 1] > 0]\n","\n","# # mask tokens\n","# test_mask_indices = torch.rand(test_input_ids.shape) < 0.15  # 15% probability\n","# test_mask_indices[:, 0] = False  # ensure first token is not masked\n","# test_mask_indices[:, -1] = False  # ensure last token is not masked\n","# masked_tokens = test_input_ids.clone()\n","# masked_tokens[test_mask_indices] = tokenizer.mask_token_id\n","# masked_tokens = torch.tensor(masked_tokens)\n","\n","# # generate test_labels from masked tokens\n","# test_labels = test_input_ids.clone()\n","# test_labels[~test_mask_indices] = -100\n","# test_labels[test_mask_indices] = test_input_ids[test_mask_indices]\n","# test_labels = torch.tensor(test_labels)\n","\n","# --------------------------------------------------------------------------- #\n","# --------------------------------------------------------------------------- #\n","\n","# --- new method --- #\n","\n","# Keep all examples (no filtering)\n","test_input_ids = test_tokenized_examples['input_ids']\n","\n","# Mask tokens with conditions\n","test_mask_indices = torch.rand(test_input_ids.shape) < 0.15  # 15% probability\n","test_mask_indices[test_tokenized_examples['attention_mask'] == 0] = False  # Exclude padding tokens\n","test_mask_indices[:, 0] = False  # Ensure first token is not masked\n","# test_mask_indices[test_input_ids == 0] = False  # Exclude tokens with value 0\n","# test_mask_indices[test_input_ids == 1] = False  # Exclude tokens with value 1\n","test_mask_indices[test_input_ids == 2] = False  # Exclude tokens with value 2\n","\n","masked_tokens = test_input_ids.clone()\n","masked_tokens[test_mask_indices] = tokenizer.mask_token_id\n","masked_tokens = torch.tensor(masked_tokens)\n","\n","# Generate test_labels from masked tokens\n","test_labels = test_input_ids.clone()\n","test_labels[~test_mask_indices] = -100\n","test_labels[test_mask_indices] = test_input_ids[test_mask_indices]\n","test_labels = torch.tensor(test_labels)"],"metadata":{"id":"hOSGPFxliYLF","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(\n","    tokenizer.eos_token,\n","    tokenizer('<s>Hi</s>'),\n","    tokenizer('Hi'),\n","    sep='\\n'\n",")"],"metadata":{"id":"uX1s-Jj4zKrX","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# test example\n","# -> https://huggingface.co/transformers/v3.0.2/model_doc/bart.html#bartforsequenceclassification\n","\n","tmp_inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\").to(device)\n","tmp_labels = torch.tensor([1]).unsqueeze(0).to(device)  # Batch size 1\n","\n","outputs = model(**tmp_inputs, labels=tmp_labels)\n","loss, logits = outputs[:2]\n","\n","print(\n","    tmp_inputs,\n","    tmp_labels,\n","    loss,\n","    sep='\\n\\n'\n",")"],"metadata":{"id":"U_e4jmpnxh98","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":[],"metadata":{"id":"gpAo42dHxju9"}},{"cell_type":"code","source":["single_batch = next(iter(dataloader))\n","display(single_batch)"],"metadata":{"id":"q56v5L24pj_J","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["outputs = model(input_ids=single_batch['inputs'], labels=single_batch['labels'])\n","loss = outputs.loss  # Access the loss output\n","logits = outputs.logits  # Access the logits output\n","\n","print(loss)"],"metadata":{"id":"g1EWWSrcr8_L","executionInfo":{"status":"aborted","timestamp":1712422497762,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["assert False"],"metadata":{"id":"XBWEuh15eZIA","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def continued_train_loop(\n","        train_dataloader,\n","        val_dataloader,\n","        model,\n","        optimizer,\n","        reporting_interval=50,\n","        save_state=False\n","    ) -> dict:\n","\n","    # --- Training Loop --- #\n","    model.train() # training mode\n","    train_epoch_loss = 0\n","    val_epoch_loss = 0\n","\n","    for batch, example in enumerate(train_dataloader):\n","\n","        inputs = {\n","            \"input_ids\": example[\"inputs\"],\n","            \"labels\": example[\"labels\"],\n","            \"attention_mask\": example[\"attention_mask\"],\n","        }\n","\n","        # forward pass\n","        outputs = model(**inputs)\n","\n","        # get loss\n","        # loss = nn.functional.masked_cross_entropy_loss(outputs.logits, inputs[\"labels\"])\n","        # loss = nn.CrossEntropyLoss()(outputs.logits, inputs[\"labels\"])\n","        # mask_positions = batch['mask_positions']\n","        loss = outputs.loss\n","\n","        # backward pass\n","        loss.backward()\n","        optimizer.step()\n","        optimizer.zero_grad()\n","\n","        train_epoch_loss += loss.item()\n","\n","        # training progress\n","        if int(batch + 1) % reporting_interval == 0:\n","            print('\\tFinished batches: ', str(batch + 1))\n","            print('\\tCurrent average loss: ', train_epoch_loss/batch)\n","\n","    # --- Validation Loop --- #\n","    model.eval()  # evaluation mode\n","    with torch.no_grad():  # disable gradient calculation\n","\n","        for batch, example in enumerate(val_dataloader):\n","            inputs = {\n","                \"input_ids\": example[\"inputs\"],\n","                \"labels\": example[\"labels\"],\n","                \"attention_mask\": example[\"attention_mask\"],\n","            }\n","            outputs = model(**inputs)\n","            loss = outputs.loss\n","\n","            val_epoch_loss += loss.item()\n","\n","    # Print training and validation loss\n","    print(f\"Average training loss: {round(train_epoch_loss/batch, 5)}\")\n","    print(f\"Average validation loss: {round(val_epoch_loss / len(val_dataloader), 5)}\")\n","\n","    if save_state:\n","        torch.save(model.state_dict(), 'states/model_state.pth')\n","\n","    return {'state':model.state_dict(), }\n","\n","def continued_train_loop(\n","        train_dataloader,\n","        val_dataloader,\n","        model,\n","        optimizer,\n","        reporting_interval=50,\n","        save_state=False\n","    ) -> dict:\n","\n","    # --- Training Loop --- #\n","    model.train()  # training mode\n","    train_epoch_loss = 0\n","    val_epoch_loss = 0\n","    val_correct = 0\n","    train_acc_history = []  # store training accuracies for each batch\n","\n","    loss_history = []  # store losses for each batch\n","\n","    for batch, example in enumerate(train_dataloader):\n","\n","        inputs = {\n","            \"input_ids\": example[\"inputs\"],\n","            \"labels\": example[\"labels\"],\n","            \"attention_mask\": example[\"attention_mask\"],\n","        }\n","\n","        # forward pass\n","        outputs = model(**inputs)\n","\n","        # get loss\n","        loss = outputs.loss\n","        loss_history.append(loss.item())  # store batch loss\n","\n","        # backward pass\n","        loss.backward()\n","        optimizer.step()\n","        optimizer.zero_grad()\n","\n","        train_epoch_loss += loss.item()\n","\n","        # get indices of masked tokens\n","        mask = (inputs[\"input_ids\"] == tokenizer.mask_token_id)  # identify masked tokens\n","\n","        # get logits and predictions for masked tokens\n","        masked_logits = outputs.logits[mask]\n","        predictions = torch.argmax(masked_logits, dim=-1)  # get predicted token index\n","\n","        # filter labels on masked tokens\n","        flat_labels = inputs[\"labels\"][mask]\n","\n","        # calculate accuracy\n","        correct = (predictions == flat_labels).sum().item()\n","        train_acc = correct / len(predictions)\n","\n","        # training progress\n","        if int(batch + 1) % reporting_interval == 0:\n","            print(f'\\tFinished batches: {str(batch + 1)} | Train Loss: {loss.item():.5f} | Train Acc: {train_acc:.5f}')\n","\n","        train_acc_history.append(train_acc)\n","\n","    # --- Validation Loop --- #\n","    model.eval()  # evaluation mode\n","    with torch.no_grad():  # disable gradient calculation\n","\n","        for batch, example in enumerate(val_dataloader):\n","            inputs = {\n","                \"input_ids\": example[\"inputs\"],\n","                \"labels\": example[\"labels\"],\n","                \"attention_mask\": example[\"attention_mask\"],  # include attention mask for validation\n","            }\n","            outputs = model(**inputs)\n","            loss = outputs.loss\n","\n","            val_epoch_loss += loss.item()\n","\n","            # calculate accuracy (validation loop)\n","            mask = (inputs[\"input_ids\"] == tokenizer.mask_token_id)\n","            masked_logits = outputs.logits[mask]\n","            predictions = torch.argmax(masked_logits, dim=-1)\n","\n","            flat_labels = inputs[\"labels\"].view(-1)\n","            masked_labels = flat_labels[mask]\n","\n","            val_correct += (predictions == masked_labels).sum().item()\n","\n","    # Print training and validation results\n","    train_acc = train_correct / (len(train_dataloader.dataset) * mask.sum().item())  # consider only masked tokens (final calculation)\n","    val_acc = val_correct / (len(val_dataloader.dataset) * mask.sum().item())  # consider only masked tokens (final calculation)\n","    print(f\"Average training loss: {round(train_epoch_loss/batch, 5)} | Train Accuracy: {train_acc:.5f}\")\n","    print(f\"Average validation loss: {round(val_epoch_loss / len(val_dataloader), 5)} | Validation Accuracy: {val_acc:.5f}\")\n","\n","    if save_state:\n","        torch.save(model.state_dict(), 'states/model_state.pth')\n","\n","    # Update return dictionary\n","    return {\n","        'state': model.state_dict(),\n","        'train_loss_history': loss_history,\n","        'train_acc_history': train_acc_history,\n","        'val_acc': val_acc\n","    }"],"metadata":{"id":"d-NDWtOm1RDe","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":9,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# define Datast class\n","# ------------------\n","\n","class ContinuedPretrainData(Dataset):\n","    def __init__(self, base_data, tokenizer, device, max_len=512):\n","        self.max_len = max_len\n","        self.tokenizer = tokenizer\n","        self.data = []\n","        self.labels = []\n","        # self.mask_positions = [] # Depracated\n","\n","        tokenized_examples = tokenizer(\n","            df_train,\n","            max_length=512,\n","            truncation=True,\n","            padding='max_length',\n","            return_tensors=\"pt\"\n","        )\n","\n","        # filter out short examples\n","        input_ids = tokenized_examples['input_ids'][tokenized_examples['attention_mask'][:, max_len - 1] > 0]\n","\n","        # mask tokens\n","        masked_indices = torch.rand(input_ids.shape) < 0.15  # 15% probability\n","        masked_indices[:, 0] = False  # ensure first token is not masked\n","        masked_indices[:, -1] = False  # ensure last token is not masked\n","        masked_tokens = input_ids.clone()\n","        masked_tokens[masked_indices] = tokenizer.mask_token_id\n","        masked_tokens = torch.tensor(masked_tokens)\n","\n","        # generate labels from masked tokens\n","        labels = input_ids.clone()\n","        labels[~masked_indices] = -100\n","        labels[masked_indices] = input_ids[masked_indices]\n","        labels = torch.tensor(labels)\n","\n","        # masked positions (depracated)\n","        # mask_positions = input_ids.clone()\n","        # mask_positions[~masked_indices] = False\n","        # mask_positions[masked_indices] = True\n","        # mask_positions = torch.tensor(mask_positions)\n","\n","        # send data to device\n","        self.data = masked_tokens.to(device)\n","        self.labels = labels.to(device)\n","        # self.mask_positions = mask_positions.to(device)\n","\n","    def __len__(self):\n","        return self.data.shape[0]\n","\n","    def __getitem__(self, index):\n","        return {'inputs': self.data[index],\n","                'labels': self.labels[index],\n","                # 'mask_positions': self.mask_positions[index],\n","        }"],"metadata":{"id":"02OAq6X8eYBr","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# define training loop\n","# ------------------\n","\n","optimizer = AdamW(model.parameters(), lr=2e-5)\n","\n","steps = 0\n","# batch_size = 4 # this doesn't work - yields an error\n","batch_size = 2048\n","\n","# training loop\n","for epoch in range(2):\n","\n","    train_dataset = ContinuedPretrainData(df_train, tokenizer, device)\n","\n","    dataloader = DataLoader(\n","        train_dataset,\n","        batch_size=batch_size,\n","        shuffle=True\n","    )\n","\n","    for batch in dataloader:\n","\n","        inputs = {\n","            \"input_ids\": batch[\"inputs\"],\n","            \"labels\": batch[\"labels\"],\n","        }\n","\n","        # Forward pass\n","        outputs = model(**inputs)\n","\n","        # Loss calculation\n","        loss = nn.CrossEntropyLoss()(outputs.logits, inputs[\"labels\"])\n","\n","        # Backward pass and optimization\n","        loss.backward()\n","        optimizer.step()\n","        optimizer.zero_grad()\n","\n","        # Print training progress (optional)\n","        if steps % 100 == 0:\n","            print(f\"Epoch: {epoch+1}/{3}, Step: {steps}, Loss: {loss.item()}\")\n","            steps += 1\n"],"metadata":{"id":"Q7qmjMZVmxS2","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataset[0]"],"metadata":{"id":"3mTwd5rLnk9E","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":8,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["num_zeros = []\n","num_twos = []\n","for i in range(0,len(train_dataset)):\n","    # count the number of times 0 appears in the example\n","    num_zeros += [torch.sum(train_dataset[i]['inputs'] == 0).item()]\n","    num_twos += [torch.sum(train_dataset[i]['inputs'] == 2).item()]\n","\n","len(num_zeros)\n"],"metadata":{"id":"bSj4UJWPlKQ3","executionInfo":{"status":"aborted","timestamp":1712422497763,"user_tz":240,"elapsed":7,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["num_zeros = np.array(num_zeros)\n","num_twos = np.array(num_twos)\n","sum(num_zeros == 1)\n","sum(num_twos == 1)"],"metadata":{"id":"09RDe0bdoGGK","executionInfo":{"status":"aborted","timestamp":1712422497766,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# decode\n","print(\n","    tokenizer.decode(train_dataset[3]['inputs']),\n","    sep='\\n',\n",")"],"metadata":{"id":"dOelIZNOT1fm","executionInfo":{"status":"aborted","timestamp":1712422497766,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"05cDpjdoT1dA","executionInfo":{"status":"aborted","timestamp":1712422497767,"user_tz":240,"elapsed":11,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"3-e7tvR4T1aZ","executionInfo":{"status":"aborted","timestamp":1712422497767,"user_tz":240,"elapsed":10,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"nOCD-tytT1Xg","executionInfo":{"status":"aborted","timestamp":1712422497771,"user_tz":240,"elapsed":14,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"9aHyPYuBT1SJ","executionInfo":{"status":"aborted","timestamp":1712422497773,"user_tz":240,"elapsed":337840,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"aoHsKVDxmxDV","executionInfo":{"status":"aborted","timestamp":1712422497773,"user_tz":240,"elapsed":337836,"user":{"displayName":"Dylan Daniels","userId":"06055247124848663483"}}},"execution_count":null,"outputs":[]}]}