Made some improvements to the code

- Removed update of today data. - Keep last and drop duplicates according to date
opendatakerala · Dec 3, 2023 · cf6d204 · cf6d204
1 parent cea2d8d
commit cf6d204
Show file tree

Hide file tree

Showing 17 changed files with 8 additions and 59 deletions.
diff --git a/.gitignore b/.gitignore
@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+activate.sh
diff --git a/daily_data_updater.py b/daily_data_updater.py
@@ -5,9 +5,11 @@
 from collections import defaultdict
 from data_update import collect_data
 from datetime import datetime, timedelta
+
 today = datetime.now()
+yesterday = today - timedelta(days = 1)
 
-days = pd.date_range(end=today, periods=4, freq='D')
+days = pd.date_range(end=yesterday, periods=7, freq='D')
 
 
 headers = {'User-Agent':"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"}
@@ -31,6 +33,5 @@
     Path(dist_path).mkdir(parents=True, exist_ok=True)
     if Path(dist_path + "/" + key + ".csv").is_file():
         old_df = pd.read_csv(dist_path + "/" + key + ".csv")
-        merged_df = pd.concat([old_df, dist_df]).drop_duplicates()
-        merged_df.to_csv(dist_path + "/" + key + ".csv", mode='w', header=True, index=None)
-
+        merged_df = pd.concat([old_df, dist_df]).drop_duplicates(['date'], keep='last')
+        merged_df.to_csv(dist_path + "/" + key + ".csv", mode='w', header=True, index=None)
diff --git a/vaccine_data/Alappuzha.csv b/vaccine_data/Alappuzha.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
 2023-11-28,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
 2023-11-29,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
-2023-11-30,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
-2023-12-01,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
-2023-12-02,0,1590258,1778987,899,1740038,1630110,267119,2994163,550676,0,0,91604,23,0,0,3637267
diff --git a/vaccine_data/Ernakulam.csv b/vaccine_data/Ernakulam.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
 2023-11-28,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
 2023-11-29,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
-2023-11-30,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
-2023-12-01,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
-2023-12-02,0,3149268,3019851,1417,3316253,2854290,373059,5547364,846472,16846,0,131623,645,0,0,6543602
diff --git a/vaccine_data/Idukki.csv b/vaccine_data/Idukki.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
 2023-11-28,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
 2023-11-29,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
-2023-11-30,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
-2023-12-01,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
-2023-12-02,0,887664,889239,478,962800,814581,74862,1563480,246672,0,0,42061,30,0,0,1852243
diff --git a/vaccine_data/Kannur.csv b/vaccine_data/Kannur.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
 2023-11-28,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
 2023-11-29,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
-2023-11-30,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
-2023-12-01,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
-2023-12-02,0,1928511,2178974,1398,2174713,1934172,209908,3737635,470242,8336,0,102489,85,0,0,4318793
diff --git a/vaccine_data/Kasaragod.csv b/vaccine_data/Kasaragod.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
 2023-11-28,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
 2023-11-29,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
-2023-11-30,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
-2023-12-01,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
-2023-12-02,0,927227,1026266,547,1060417,893625,64298,1690053,272545,0,0,55742,0,0,0,2018340
diff --git a/vaccine_data/Kollam.csv b/vaccine_data/Kollam.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
 2023-11-28,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
 2023-11-29,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
-2023-11-30,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
-2023-12-01,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
-2023-12-02,0,1925016,2262977,1335,2234268,1955063,236644,3675846,645059,4803,0,99369,485,0,0,4425975
diff --git a/vaccine_data/Kottayam.csv b/vaccine_data/Kottayam.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
 2023-11-28,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
 2023-11-29,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
-2023-11-30,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
-2023-12-01,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
-2023-12-02,0,1560621,1617206,883,1682091,1496624,200579,2916376,379956,1179,0,80908,147,0,0,3379294
diff --git a/vaccine_data/Kozhikode.csv b/vaccine_data/Kozhikode.csv
@@ -995,6 +995,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
 2023-11-28,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
 2023-11-29,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
-2023-11-30,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
-2023-12-01,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
-2023-12-02,0,2350964,2603440,1657,2703909,2252154,203208,4495408,559206,4808,0,99574,275,0,0,5159271
diff --git a/vaccine_data/Malappuram.csv b/vaccine_data/Malappuram.csv
@@ -996,7 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,1,2913733,3279865,2478,3466900,2729177,190946,5435152,775066,6562,0,170033,119,0,0,6387023
 2023-11-28,3,2913734,3279866,2478,3466900,2729179,190947,5435152,775066,6562,0,170033,121,0,0,6387026
 2023-11-29,1,2913735,3279866,2478,3466901,2729179,190947,5435152,775066,6562,0,170033,122,0,0,6387027
-2023-11-30,0,2913735,3279866,2478,3466901,2729179,190947,5435152,775066,6562,0,170033,122,0,0,6387027
-2023-12-01,1,2913735,3279866,2478,3466901,2729179,190948,5435152,775066,6562,0,170033,122,0,0,6387028
-2023-12-02,0,2913735,3279866,2478,3466901,2729179,190948,5435152,775066,6562,0,170033,122,0,0,6387028
-2023-12-02,3,2913736,3279868,2478,3466904,2729179,190948,5435152,775066,6562,0,170033,125,0,0,6387031
diff --git a/vaccine_data/Palakkad.csv b/vaccine_data/Palakkad.csv
@@ -996,8 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,7,2052978,2335255,1334,2346196,2043373,254196,3944105,589678,2864,0,106292,423,0,0,4643765
 2023-11-28,0,2052978,2335256,1334,2346196,2043374,254196,3944105,589678,2864,0,106292,424,0,0,4643766
 2023-11-29,6,2052978,2335257,1334,2346197,2043374,254201,3944105,589678,2864,0,106292,425,0,0,4643772
-2023-11-30,0,2052978,2335257,1334,2346197,2043374,254207,3944105,589678,2864,0,106292,425,0,0,4643778
-2023-12-01,6,2052978,2335261,1334,2346201,2043374,254209,3944105,589678,2864,0,106292,429,0,0,4643784
-2023-12-02,0,2052978,2335261,1334,2346201,2043374,254209,3944105,589678,2864,0,106292,429,0,0,4643784
-2023-12-02,0,2052978,2335261,1334,2346201,2043374,254212,3944105,589678,2864,0,106292,429,0,0,4643787
-2023-12-02,0,2052978,2335263,1334,2346201,2043376,254212,3944105,589678,2864,0,106292,431,0,0,4643789
diff --git a/vaccine_data/Pathanamthitta.csv b/vaccine_data/Pathanamthitta.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
 2023-11-28,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
 2023-11-29,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
-2023-11-30,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
-2023-12-01,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
-2023-12-02,0,1021581,1132306,456,1158681,995663,186172,2024308,273830,0,0,42323,55,0,0,2340516
diff --git a/vaccine_data/Thiruvananthapuram.csv b/vaccine_data/Thiruvananthapuram.csv
@@ -996,9 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,5,2565840,2869800,1427,2901040,2536036,340446,4842123,788794,22932,0,121821,1367,0,0,5777522
 2023-11-28,1,2565841,2869800,1427,2901040,2536037,340446,4842123,788794,22932,0,121821,1368,0,0,5777523
 2023-11-29,7,2565843,2869804,1427,2901044,2536039,340447,4842123,788794,22932,0,121821,1374,0,0,5777530
-2023-11-30,4,2565845,2869808,1427,2901047,2536043,340448,4842123,788794,22932,0,121821,1381,0,0,5777538
-2023-12-01,1,2565845,2869809,1427,2901047,2536044,340448,4842123,788794,22932,0,121821,1382,0,0,5777539
-2023-12-01,4,2565846,2869809,1427,2901047,2536045,340450,4842123,788794,22932,0,121821,1383,0,0,5777542
-2023-12-02,0,2565846,2869809,1427,2901047,2536045,340450,4842123,788794,22932,0,121821,1383,0,0,5777542
-2023-12-02,5,2565846,2869809,1427,2901047,2536045,340452,4842123,788794,22932,0,121821,1383,0,0,5777544
-2023-12-02,9,2565846,2869812,1427,2901049,2536046,340456,4842123,788794,22932,0,121821,1386,0,0,5777551
diff --git a/vaccine_data/Thrissur.csv b/vaccine_data/Thrissur.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
 2023-11-28,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
 2023-11-29,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
-2023-11-30,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
-2023-12-01,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
-2023-12-02,0,2404447,2727890,1440,2652104,2481676,429471,4728144,691162,4494,0,138857,137,0,0,5563251
diff --git a/vaccine_data/Wayanad.csv b/vaccine_data/Wayanad.csv
@@ -996,6 +996,3 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-27,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
 2023-11-28,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
 2023-11-29,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
-2023-11-30,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
-2023-12-01,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
-2023-12-02,0,684466,723582,388,752376,656060,57102,1212879,213894,0,0,38765,0,0,0,1465538
diff --git a/vaccine_data/kerala.csv b/vaccine_data/kerala.csv
@@ -994,10 +994,4 @@ date,daily,male,female,others,Total dose 1,Total dose 2,Total Precuation dose,co
 2023-11-26,0,25962566,28445633,16137,29151782,25272595,3088005,48807036,7303252,72824,0,1321461,3778,0,0,57512382
 2023-11-27,13,25962574,28445638,16137,29151786,25272604,3088010,48807036,7303252,72824,0,1321461,3791,0,0,57512400
 2023-11-28,4,25962576,28445640,16137,29151786,25272608,3088011,48807036,7303252,72824,0,1321461,3795,0,0,57512405
-2023-11-29,14,25962579,28445645,16137,29151792,25272610,3088017,48807036,7303252,72824,0,1321461,3803,0,0,57512419
-2023-11-30,4,25962581,28445649,16137,29151795,25272614,3088024,48807036,7303252,72824,0,1321461,3810,0,0,57512433
-2023-12-01,8,25962581,28445654,16137,29151799,25272615,3088027,48807036,7303252,72824,0,1321461,3815,0,0,57512441
-2023-12-01,11,25962582,28445654,16137,29151799,25272616,3088029,48807036,7303252,72824,0,1321461,3816,0,0,57512444
-2023-12-02,0,25962582,28445654,16137,29151799,25272616,3088029,48807036,7303252,72824,0,1321461,3816,0,0,57512444
-2023-12-02,5,25962582,28445654,16137,29151799,25272616,3088034,48807036,7303252,72824,0,1321461,3816,0,0,57512449
-2023-12-02,12,25962583,28445661,16137,29151804,25272619,3088038,48807036,7303252,72824,0,1321461,3824,0,0,57512461
+2023-11-29,14,25962579,28445645,16137,29151792,25272610,3088017,48807036,7303252,72824,0,1321461,3803,0,0,57512419