Merge pull request #390 from onc-healthit/LANTERN-579-Fix-eMedPractic…

…eWebscraper

Lantern 579 Fix eMedPractice webscraper

endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go

@@ -64,7 +64,7 @@ var correctekURL = "https://ulrichmedicalconcepts.com/home/the-ehr/meaningful-us
 var varianmedicalURL = "https://variandev.dynamicfhir.com/"
 var caretrackerURL = "https://hag-fhir.amazingcharts.com/ac/endpoints"
 var zhhealthcareURL = "https://blueehr.com/fhir-urls/"
-var emedpracticeURL = "https://emedpractice.com/Fhir/FhirHelpDocument.html"
+var emedpracticeURL = "https://emedpractice.com/fhir/fhirhelpdocument.html"
 var doc_torURL = "https://hag-fhir.amazingcharts.com/pc/endpoints"
 var azaleahealthURL = "https://api.azaleahealth.com/fhir/R4/Endpoint"
 var cloudcraftURL = "https://fhirapitest.naiacorp.net/fhir/r4/endpoints/"
@@ -288,7 +288,7 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 	} else if chplURL == medinfoengineeringURL {
 		MedicalInformaticsEngineeringWebscraper(chplURL, fileToWriteTo)
 	} else if chplURL == emedpracticeURL {
-		eMedPracticeWebscraper("https://servicebackup.emedpractice.com:8443/helpdoc/fhir_helpdoc.html", fileToWriteTo)
+		eMedPracticeWebscraper(chplURL, fileToWriteTo)
 	} else if chplURL == doc_torURL {
 		BundleQuerierParser(chplURL+"/r4", fileToWriteTo)
 	} else if URLsEqual(chplURL, azaleahealthURL) {

endpointmanager/pkg/chplendpointquerier/emedpracticewebscraper.go

@@ -2,40 +2,67 @@ package chplendpointquerier
 
 import (
 	"strings"
+	"fmt"
 
 	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
 	log "github.com/sirupsen/logrus"
+	"github.com/PuerkitoBio/goquery"
+
 )
 
 func eMedPracticeWebscraper(CHPLURL string, fileToWriteTo string) {
 
 	var lanternEntryList []LanternEntry
 	var endpointEntryList EndpointList
 
-	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, "#fhir-api-documentation")
+	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, "#bulkDataExport")
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	apiDocElem := doc.Find("#fhir-api-documentation")
-	if apiDocElem.Length() > 0 {
-		pElemURL := apiDocElem.Eq(0).Next().Next()
-		codeElemURL := pElemURL.Find("code")
-		codeElemText := codeElemURL.Eq(0).Text()
+	urlFound := false
 
-		var entry LanternEntry
+	doc.Find("ul").Each(func(index int, ulElems *goquery.Selection) {
+		ulElems.Find("li").Each(func(index int, liElems *goquery.Selection) {
+			if strings.Contains(liElems.Text(), "https:") && !urlFound {
+				divElems := liElems.Find("div")
+				if divElems.Length() > 0 {
+					preElems := liElems.Find("pre")
+					if preElems.Length() > 0 {
+						var entry LanternEntry
+
+						urlStart := strings.Index(preElems.Text(), "https://")
+						if urlStart == -1 {
+							fmt.Println("URL not found")
+							return
+						}
 
-		entryURL := strings.TrimSpace(codeElemText)
-		entry.URL = entryURL
+						urlEnd := strings.Index(preElems.Text()[urlStart:], "8443/")
+						if urlEnd == -1 {
+							fmt.Println("End of base URL not found")
+							return
+						}
 
-		lanternEntryList = append(lanternEntryList, entry)
-	}
+						fhirURL := preElems.Text()[urlStart : urlStart+urlEnd+len("8443")]
+						entry.URL = fhirURL
+						lanternEntryList = append(lanternEntryList, entry)
+
+						urlFound = true
+
+						return
+					}
+				}
+			}
+		})
 
-	endpointEntryList.Endpoints = lanternEntryList
+		if urlFound {
+			return
+		}
+	})
 
+	endpointEntryList.Endpoints = lanternEntryList
 	err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
 	if err != nil {
 		log.Fatal(err)
 	}
-
 }