From 506bbd45a81899098dfdf908b59530e6911bed67 Mon Sep 17 00:00:00 2001 From: archita-ekkirala Date: Wed, 18 Sep 2024 18:26:59 -0500 Subject: [PATCH 1/3] LANTERN-575: 1up CSV Parser --- .../pkg/chplendpointquerier/csvparser.go | 38 +++++++--- .../endpointwebscraper/endpointwebscraper.go | 4 +- resources/prod_resources/oneupdata.csv | 72 +++++++++++++++++++ 3 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 resources/prod_resources/oneupdata.csv diff --git a/endpointmanager/pkg/chplendpointquerier/csvparser.go b/endpointmanager/pkg/chplendpointquerier/csvparser.go index 5db80de15..4f6268439 100644 --- a/endpointmanager/pkg/chplendpointquerier/csvparser.go +++ b/endpointmanager/pkg/chplendpointquerier/csvparser.go @@ -1,6 +1,7 @@ package chplendpointquerier import ( + "encoding/csv" "io" "strings" @@ -11,15 +12,34 @@ import ( log "github.com/sirupsen/logrus" ) -func CSVParser(CHPLURL string, fileToWriteTo string, csvFilePath string, numrecords int, startrecord int, header bool, urlIndex int, organizationIndex int) { +func CSVParser(inputSource string, fileToWriteTo string, csvFilePath string, numrecords int, startrecord int, header bool, urlIndex int, organizationIndex int) { var lanternEntryList []LanternEntry var endpointEntryList EndpointList - csvReader, file, err := helpers.QueryAndOpenCSV(CHPLURL, csvFilePath, header) - if err != nil { - log.Fatal(err) + var csvReader *csv.Reader + var file *os.File + var err error + if strings.HasPrefix(inputSource, "http://") || strings.HasPrefix(inputSource, "https://") { + csvReader, file, err = helpers.QueryAndOpenCSV(inputSource, csvFilePath, header) + if err != nil { + log.Fatal(err) + } + defer file.Close() + } else { + file, err = os.Open(inputSource) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + csvReader = csv.NewReader(file) + if header { + _, err := csvReader.Read() + if err != nil { + log.Fatal(err) + } + } } - defer file.Close() records := 0 for { @@ -59,8 +79,10 @@ func CSVParser(CHPLURL string, fileToWriteTo string, csvFilePath string, numreco log.Fatal(err) } - err = os.Remove(csvFilePath) - if err != nil { - log.Fatal(err) + if _, err := os.Stat(csvFilePath); err == nil { + err = os.Remove(csvFilePath) + if err != nil { + log.Fatal(err) + } } } diff --git a/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go b/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go index e3ab4e6d7..e57308566 100644 --- a/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go +++ b/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go @@ -3,6 +3,8 @@ package endpointwebscraper import ( "encoding/json" "os" + + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/chplendpointquerier" ) type EndpointList struct { @@ -24,7 +26,7 @@ func EndpointListWebscraper(vendorURL string, vendor string, fileToWriteTo strin if vendorURL == careEvolutionURL { HTMLtablewebscraper(vendorURL, vendor, fileToWriteTo) } else if vendorURL == oneUpURL { - OneUpQuerier("https://api.1up.health/connect/system/clinical", fileToWriteTo) + chplendpointquerier.CSVParser("../../../resources/prod_resources/oneupdata.csv", "1UpEndpointSources.json", "", -1, 0, true, 2, 1) } } diff --git a/resources/prod_resources/oneupdata.csv b/resources/prod_resources/oneupdata.csv new file mode 100644 index 000000000..aa4c12054 --- /dev/null +++ b/resources/prod_resources/oneupdata.csv @@ -0,0 +1,72 @@ +Parent Organization Name,"Brand Name(s) (displayed in auth app, recognizable by members)",Prod FHIR URL (Resource URL),Notes +AHF,AIDS Healthcare Foundation (AHF),https://api.ahffhir.com/r4, +AHF,Positive Healthcare,https://api.ahffhir.com/r4, +Arkansas,Arkansas Medicaid,https://api.arkansasfhir.com/r4, +Centers Health Plan,Centers Plan Healthy Living (CPHL),https://api.cphlfhir.com/r4, +Cox HealthPlans,Cox HealthPlans,https://api.coxhealthplanfhir.com/r4, +Dean Health Plan,Dean Health Plan,https://api.deanhealthfhir.com/r4, +Dean Health Plan,WellFirst Health (WFH),https://api.deanhealthfhir.com/r4, +Dean Health Plan,WFH provided by SSM Health Plan (WFH SSM),https://api.deanhealthfhir.com/r4, +Dean Health Plan,PREVEA360 (P360),https://api.deanhealthfhir.com/r4, +Delaware State Medicaid,Delaware Medicaid,https://api.delawarefhir.com/r4, +Hamaspik Choice,Hamaspik Choice,https://api.hamaspikfhir.com/r4, +GlobalHealth,GlobalHealth,https://api.globalhealthfhir.com/r4, +HealthPartners,HealthPartners,https://api/healthpartnersfhir.com/rt, +Harvard Pilgrim,Harvard Pilgrim,https://api.hphcfhir.com/r4, +Southwestern Health,Care N' Care,https://api.swhrfhir.com/r4, +Health Plan San Mateo (HPSM),Health Plan San Mateo (HPSM),https://api.hpsmfhir.com/r4, +Inclusa,Inclusa,https://api.inclusafhir.com/r4, +Kentucky,Kentucky Medicaid,https://api.kentuckyfhir.com/r4, +Network Health,Network Health,https://api.networkhealthfhir.com/r4, +Oscar,Oscar,https://api.oscarfhir.com/r4, +Partners Health Management,Partners Health Management,https://api.partnershealthfhir.com/r4, +State of Kansas,Kansas Medical Assistance Program (KMAP),https://api.kansasfhir.com/r4, +Vaya Health,Vaya Health,https://api.vayahealthfhir.com/r4, +Visiting Nurse Service of New York (VNSNY),VNSNY,https://api.vnshealthfhir.com/r4, +Visiting Nurse Service of New York (VNSNY),VNSNY CHOICE,https://api.vnshealthfhir.com/r4, +Western Health,Western Health Advantage,https://api.westernhealthfhir.com/r4, +Zing Health,Zing Health,https://api.zinghealthfhir.com/r4, +Boston Medical Center (BMC) Health,"BMC HealthNet Plan +Well Sense Health Plan",https://api.bmcfhir.com/r4, +Georgia,Georgie Medicaid,https://api.georgiafhir.com/r4, +Maine,MaineCare,https://api.mainefhir.com/r4, +Nevada,Nevada Medicaid,https://api.nevadafhir.com/r4, +Oklahoma,Oklahoma Health Care Authority,https://api.oklahomafhir.com/r4, +Rhode Island,Rhode Island Medicaid,https://api.rhodeislandfhir.com/r4, +State of Louisiana,Louisiana Medicaid,https://api.louisianafhir.com/r4, +Tufts Health Plan,Tufts Health Public Plans (THPP),https://api.tuftsfhir.com/r4, +Tufts Health Plan,CarePartners of CT (CPCT),https://api.tuftsfhir.com/r4, +Tufts Health Plan,Tufts Senior Plan (Medicare),https://api.tuftsfhir.com/r4, +West Virginia,West Viriginia Medicaid,https://api.westvirginiafhir.com/r4, +Wisconsin,Wisconsin Department of Health Services,https://api.wisconsinfhir.com/r4, +Cook County Health,Cook County Health CountyCare ,https://api.cookcountyhealthfhir.com/r4, +Priority Health,,https://api.priorityfhir.com/r4, +Empower Health,Empower Health,https://api.empowerfhir.com/r4, +Physicians Health Plan,,https://api.phpfhir.com/r4, +AultCare,AultCare,https://api.aultcarefhir.com/r4, +Alaska,Alaska Department of Health,https://api.alaskafhir.com/r4, +Georgia,Georgia Department of Community Health,https://api.georgiafhir.com/r4, +Health First Health Plans,Health First ,https://api.healthfirstfhir.com/r4, +Mass General Brigham Health Plan ,MGB Health Plan,https://api.mgbhealthplanfhir.com/r4, +Fallon Health,Fallon Health,https://api.fallonhealthfhir.com/r4, +Maryland Physicians Care,Maryland Physicians Care,https://api.mpcfhir.com/r4, +MassHealth,MassHealth,https://api.masshealthfhir.com/r4, +New York Meidcaid,,, +Vermont Medicaid ,Vermont Medicaid,https://api.vermontfhir.com/r4, +Blue Cross Blue Shield Nebraska,Blue Cross Blue Shield Nebraska,https://api.bcbsnefhir.com/r4, +The Health Plan of West Virginia,The Health Plan ,https://api.thehealthplanfhir.com/r4, +MMM Healthcare,MMM Healthcare,https://api.mmmhealthcarefhir.com/r4, +MMM Healthcare,MMM Multihealth,https://api.mmmhealthcarefhir.com/r4, +Sendero Health Plans,Sendero Health Plans,https://api.senderohealthfhir.com/r4, +Cascade Health Alliance,Cascade Health Alliance,https://api.cascadecompfhir.com/r4, +Iowa Health and Human Service,Iowa Health and Human Service,https://api.iowafhir.com/r4, +Oklahoma Health Care Authority ,Oklahoma Health Care Authority ,https://api.oklahomafhir.com/r4, +Viva Health,Viva Health,https://api.vivahealthfhir.com/r4, +Medical Card System,Medical Card System,https://api.mcsfhir.com/r4, +Yamhill Community Care,Yamhill Community Care,https://api.yamhillfhir.com/r4, +Independent Living System,Florida Community Care,https://api.ilshealthfhir.com/r4/, +Independent Living System,Florida Complete Care,https://api.ilshealthfhir.com/r4/, +Commonwealth of PA (CoPa),Commonwealth of PA (CoPA),https://api.pennsylvaniafhir.com/r4, +Alliant Health Plans,Alliant Health Plans,https://api.allianthealthplansfhir.com/r4, +BayCare,BayCare,https://api.baycarefhir.com/r4, +UCare,UCare,https://api.ucarefhir.com/r4, From 9495a3f280df859eb7c1f7ccfeb145c3fbab17f8 Mon Sep 17 00:00:00 2001 From: archita-ekkirala Date: Thu, 19 Sep 2024 00:57:13 -0500 Subject: [PATCH 2/3] LANTERN-575: Fixing organization name index --- endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go b/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go index e57308566..9132a89ca 100644 --- a/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go +++ b/endpointmanager/pkg/endpointwebscraper/endpointwebscraper.go @@ -26,7 +26,7 @@ func EndpointListWebscraper(vendorURL string, vendor string, fileToWriteTo strin if vendorURL == careEvolutionURL { HTMLtablewebscraper(vendorURL, vendor, fileToWriteTo) } else if vendorURL == oneUpURL { - chplendpointquerier.CSVParser("../../../resources/prod_resources/oneupdata.csv", "1UpEndpointSources.json", "", -1, 0, true, 2, 1) + chplendpointquerier.CSVParser("../../../resources/prod_resources/oneupdata.csv", "1UpEndpointSources.json", "", -1, 0, true, 2, 0) } } From 32ceb7003b9528c6b46588105014f512bfb50f45 Mon Sep 17 00:00:00 2001 From: archita-ekkirala Date: Thu, 19 Sep 2024 01:11:35 -0500 Subject: [PATCH 3/3] LANTERN-575: Adding comments to the method --- endpointmanager/pkg/chplendpointquerier/csvparser.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/endpointmanager/pkg/chplendpointquerier/csvparser.go b/endpointmanager/pkg/chplendpointquerier/csvparser.go index 4f6268439..61b9f68b0 100644 --- a/endpointmanager/pkg/chplendpointquerier/csvparser.go +++ b/endpointmanager/pkg/chplendpointquerier/csvparser.go @@ -12,6 +12,16 @@ import ( log "github.com/sirupsen/logrus" ) +// CSVParser reads a CSV from a URL or file and writes processed data to an output file. +// Parameters: +// - inputSource: URL or file path of the input CSV. +// - fileToWriteTo: File path to write the processed data. +// - csvFilePath: Temporary file path for storing the downloaded CSV (if applicable). +// - numrecords: Number of records to process (-1 for all records). +// - startrecord: Starting index of records to process. +// - header: Boolean indicating if the CSV has a header to skip. +// - urlIndex: Column index where the URL is located. +// - organizationIndex: Column index where the organization name is located. func CSVParser(inputSource string, fileToWriteTo string, csvFilePath string, numrecords int, startrecord int, header bool, urlIndex int, organizationIndex int) { var lanternEntryList []LanternEntry var endpointEntryList EndpointList