diff --git a/install/install-tk.iss b/install/install-tk.iss index cd14575a8..4a4830609 100644 --- a/install/install-tk.iss +++ b/install/install-tk.iss @@ -3,11 +3,11 @@ ; AppID can never be changed as subsequent installations require the same installation ID each time AppID=FHIRToolkit AppName=Health Intersections FHIR Toolkit -AppVerName=FHIRToolkit v3.4.9 +AppVerName=FHIRToolkit v3.4.10 ; compilation control OutputDir=..\install\build -OutputBaseFilename=fhirtoolkit-win64-3.4.9 +OutputBaseFilename=fhirtoolkit-win64-3.4.10 Compression=lzma2/ultra64 ; 64 bit @@ -32,11 +32,11 @@ UninstallFilesDir={app}\uninstall ; win2000+ add/remove programs support AppPublisher=Health Intersections P/L AppPublisherURL=http://www.healthintersections.com.au -AppVersion=3.4.9 +AppVersion=3.4.10 AppSupportURL=https://github.com/grahamegrieve/fhirserver AppUpdatesURL=https://github.com/grahamegrieve/fhirserver AppCopyright=Copyright (c) Health Intersections Pty Ltd 2020+ -VersionInfoVersion=3.4.9.0 +VersionInfoVersion=3.4.10.0 ; dialog support LicenseFile=..\license diff --git a/install/install.iss b/install/install.iss index 6506a2320..092309968 100644 --- a/install/install.iss +++ b/install/install.iss @@ -3,11 +3,11 @@ ; AppID can never be changed as subsequent installations require the same installation ID each time AppID=FHIRServer AppName=Health Intersections FHIR Server -AppVerName=FHIRServer v3.4.9 +AppVerName=FHIRServer v3.4.10 ; compilation control OutputDir=..\install\build -OutputBaseFilename=fhirserver-win64-3.4.9 +OutputBaseFilename=fhirserver-win64-3.4.10 Compression=lzma2/ultra64 ; 64 bit @@ -34,11 +34,11 @@ UninstallFilesDir={app}\uninstall ; win2000+ add/remove programs support AppPublisher=Health Intersections P/L AppPublisherURL=http://www.healthintersections.com.au -AppVersion=3.4.9 +AppVersion=3.4.10 AppSupportURL=https://github.com/grahamegrieve/fhirserver AppUpdatesURL=https://github.com/grahamegrieve/fhirserver AppCopyright=Copyright (c) Health Intersections Pty Ltd 2011+ -VersionInfoVersion=3.4.9.0 +VersionInfoVersion=3.4.10.0 ; dialog support LicenseFile=..\license diff --git a/library/fsl/fsl_npm.pas b/library/fsl/fsl_npm.pas index 1e6f4f32e..2443111f5 100644 --- a/library/fsl/fsl_npm.pas +++ b/library/fsl/fsl_npm.pas @@ -589,7 +589,7 @@ function TNpmPackage.GetDependencies: TArray; begin sl := TStringList.Create; try - if (info.has('dependencies')) then + if (info.has('dependencies')) and (info.obj['dependencies'] <> nil) then begin for n in info.obj['dependencies'].properties.keys do sl.add(n+'#'+info.obj['dependencies'].str[n]); @@ -1114,8 +1114,11 @@ procedure TNpmPackage.readStream(tgz: TStream; desc: String; progress: TWorkProg bs.free; end; try - FNpm := TJsonParser.parse(folders['package'].fetchFile('package.json')); - except + if folders['package'] <> nil then + FNpm := TJsonParser.parse(folders['package'].fetchFile('package.json')) + else + raise EFslException.create('Error parsing '+desc+'#'+'package/package.json: Not found'); + except on e : Exception do raise EFslException.create('Error parsing '+desc+'#'+'package/package.json: '+e.Message); end; diff --git a/library/fsl/fsl_utilities.pas b/library/fsl/fsl_utilities.pas index 25f2721a5..49c39bcbc 100644 --- a/library/fsl/fsl_utilities.pas +++ b/library/fsl/fsl_utilities.pas @@ -253,6 +253,7 @@ function clength(b : TBytes) : cardinal; overload; Function StringArrayExists(Const aNames : Array Of String; Const sName: String) : Boolean; Overload; Function StringGet(Const sValue : String; iIndex : Integer) : Char; Overload; Function StringStartsWith(Const sValue, sFind : String; sensitive : boolean = false) : Boolean; Overload; +Function StringStartsWith(Const sValue : String; aFind : Array of String; sensitive : boolean = false) : Boolean; Overload; Function StringStartsWithSensitive(Const sValue, sFind : String) : Boolean; Overload; Function StringStartsWithInsensitive(Const sValue, sFind : String) : Boolean; Overload; Function StringEndsWith(Const sValue, sFind : String; sensitive : boolean = false) : Boolean; Overload; @@ -5484,6 +5485,15 @@ class procedure TFileLauncher.Open(const FilePath: string); End; +Function StringStartsWith(Const sValue : String; aFind : Array of String; sensitive : boolean = false) : Boolean; +var + s : String; +begin + result := false; + for s in aFind do + if StringStartsWith(sValue, s, sensitive) then + exit(true); +end; Function StringEqualsSensitive(Const sA, sB : String) : Boolean; Begin @@ -8029,7 +8039,7 @@ class function TFslDateTime.fromFormat(format, date: String; AllowBlankTimes: Bo else if (s = 'dec') or (s = 'december') then Result.FMonth := 12 else - raise EDateFormatError.Create('The Month "' + s + '" is unknown in '+date); + raise EDateFormatError.Create('The Month "' + s + '" is unknown in '+date+' (using format '+format+', from '+inttostr(start)+' for '+inttostr(length)+' chars'); end else if s = '' then Result.FMonth := 1 diff --git a/library/version.inc b/library/version.inc index 0565d20c5..d0d4df712 100644 --- a/library/version.inc +++ b/library/version.inc @@ -1,3 +1,3 @@ - FHIR_CODE_FULL_VERSION = '3.4.9'; - FHIR_CODE_RELEASE_DATE = '2024-08-26'; - FHIR_CODE_RELEASE_DATETIME = '20240826221403.486Z'; + FHIR_CODE_FULL_VERSION = '3.4.10'; + FHIR_CODE_RELEASE_DATE = '2024-09-05'; + FHIR_CODE_RELEASE_DATETIME = '20240905110448.596Z'; diff --git a/server/endpoint_packages.pas b/server/endpoint_packages.pas index 3ab9f1661..105aa8ca9 100644 --- a/server/endpoint_packages.pas +++ b/server/endpoint_packages.pas @@ -68,6 +68,8 @@ TPackageUpdaterThread = class(TFslThread) TMatchTableSort = (mtsNull, mtsId, mtsVersion, mtsDate, mtsFhirVersion, mtsCanonical, mtsDownloads, mtsKind); + { TFHIRPackageWebServer } + TFHIRPackageWebServer = class (TFhirWebServerEndpoint) private FDB : TFDBManager; @@ -75,6 +77,7 @@ TFHIRPackageWebServer = class (TFhirWebServerEndpoint) FNextScan : TDateTIme; FScanning: boolean; FSystemToken : String; + FCrawlerLog : String; procedure setDB(value : TFDBManager); function status : String; @@ -92,12 +95,14 @@ TFHIRPackageWebServer = class (TFhirWebServerEndpoint) procedure serveSearch(name, canonicalPkg, canonicalUrl, FHIRVersion, dependency, sort : String; secure : boolean; request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo); procedure serveUpdates(date : TFslDateTime; secure : boolean; response : TIdHTTPResponseInfo); procedure serveProtectForm(request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo; id : String); + procedure serveLog(request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo; id : String); procedure serveUpload(request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo; secure : boolean; id : String); procedure processProtectForm(request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo; id, pword : String); procedure SetScanning(const Value: boolean); function doRequest(AContext: TIdContext; request: TIdHTTPRequestInfo; response: TIdHTTPResponseInfo; id: String; secure: boolean): String; public + constructor Create(code, path : String; common : TFHIRWebServerCommon); override; destructor Destroy; override; function link : TFHIRPackageWebServer; overload; function description : String; override; @@ -416,6 +421,7 @@ procedure TPackageUpdaterThread.RunUpdater; try upd := TPackageUpdater.Create(FZulip.link); try + upd.CrawlerLog := TFslStringBuilder.create; upd.OnSendEmail := doSendEmail; try upd.update(conn); @@ -431,6 +437,7 @@ procedure TPackageUpdaterThread.RunUpdater; Logging.log('Exception updating packages: '+e.Message); end; end; + FEndPoint.FPackageServer.FCrawlerLog := upd.CrawlerLog.AsString; finally upd.free; end; @@ -804,6 +811,23 @@ procedure TFHIRPackageWebServer.serveProtectForm(request : TIdHTTPRequestInfo; r end; end; +procedure TFHIRPackageWebServer.serveLog(request: TIdHTTPRequestInfo; response: TIdHTTPResponseInfo; id: String); +var + vars : TFslMap; +begin + response.ResponseNo := 200; + response.ResponseText := 'OK'; + vars := TFslMap.Create; + try + vars.add('prefix', TFHIRObjectText.Create(AbsoluteUrl(false))); + vars.add('ver', TFHIRObjectText.Create('4.0.1')); + vars.add('log', TFHIRObjectText.Create(FCrawlerLog)); + returnFile(request, response, nil, request.Document, 'packages-log.html', false, vars); + finally + vars.free; + end; +end; + function codeForKind(kind : integer): String; begin case TFHIRPackageKind(kind) of @@ -914,7 +938,9 @@ function sel(this, that : String) : string; result := ''; end; -procedure TFHIRPackageWebServer.serveSearch(name, canonicalPkg, canonicalURL, FHIRVersion, dependency, sort : String; secure : boolean; request : TIdHTTPRequestInfo; response : TIdHTTPResponseInfo); +procedure TFHIRPackageWebServer.serveSearch(name, canonicalPkg, canonicalUrl, + FHIRVersion, dependency, sort: String; secure: boolean; + request: TIdHTTPRequestInfo; response: TIdHTTPResponseInfo); var conn : TFDBConnection; json : TJsonArray; @@ -1346,6 +1372,10 @@ function TFHIRPackageWebServer.doRequest(AContext: TIdContext; request: TIdHTTPR result := 'Packages updates since '+pm['lastUpdated']; end; end + else if (request.CommandType = hcGET) and (request.Document = '/packages/log') then + begin + serveLog(request, response, pm['id']); + end else if (request.CommandType = hcGET) and (request.Document = '/packages/protect') then begin serveProtectForm(request, response, pm['id']); @@ -1403,6 +1433,12 @@ function TFHIRPackageWebServer.doRequest(AContext: TIdContext; request: TIdHTTPR end; end; +constructor TFHIRPackageWebServer.Create(code, path: String; common: TFHIRWebServerCommon); +begin + inherited Create(code, path, common); + FCrawlerLog := 'The Crawler has not yet completed processing the feed'; +end; + function TFHIRPackageWebServer.SecureRequest(AContext: TIdContext; ip : String; request: TIdHTTPRequestInfo; response: TIdHTTPResponseInfo; cert: TIdOpenSSLX509; id: String; tt : TTimeTracker): String; begin countRequest; diff --git a/server/fhirconsole.lpi b/server/fhirconsole.lpi index 457d26b96..e02071932 100644 --- a/server/fhirconsole.lpi +++ b/server/fhirconsole.lpi @@ -18,7 +18,7 @@ - + diff --git a/server/fhirserver.dproj b/server/fhirserver.dproj index d4cc9aa48..2635b6ccc 100644 --- a/server/fhirserver.dproj +++ b/server/fhirserver.dproj @@ -92,7 +92,7 @@ true 3 4 - 9 + 10 false @@ -167,7 +167,7 @@ true false 3 - 9 + 10 none 4 false diff --git a/server/fhirserver.lpi b/server/fhirserver.lpi index 21a1e17e4..19e1aabc4 100644 --- a/server/fhirserver.lpi +++ b/server/fhirserver.lpi @@ -20,7 +20,7 @@ - + diff --git a/server/package_spider.pas b/server/package_spider.pas index a2bd00a11..cf7d3dff2 100644 --- a/server/package_spider.pas +++ b/server/package_spider.pas @@ -80,6 +80,10 @@ TZulipTracker = class (TFslObject) procedure send; end; + TCrawlerLogMode = (clmStart, clmHeader, clmError, clmWarning, clmNote); + + { TPackageUpdater } + TPackageUpdater = class (TFslObject) private FDB : TFDBConnection; @@ -89,6 +93,9 @@ TPackageUpdater = class (TFslObject) FTotalBytes : Cardinal; FIni : TIniFile; FZulip : TZulipTracker; + FCrawlerLog : TFslStringBuilder; + FHasErrors : boolean; + procedure clog(s : String; mode : TCrawlerLogMode); procedure DoSendEmail(dest, subj, body : String); procedure log(msg, source : String; error : boolean); @@ -97,6 +104,7 @@ TPackageUpdater = class (TFslObject) function fetchXml(url : string) : TMXmlElement; function hasStored(guid : String) : boolean; + procedure SetCrawlerLog(AValue: TFslStringBuilder); procedure store(source, url, guid : String; date : TFslDateTime; package : Tbytes; idver : String); procedure updateItem(source : String; item : TMXmlElement; i : integer; pr : TPackageRestrictions); @@ -108,6 +116,7 @@ TPackageUpdater = class (TFslObject) procedure update(DB : TFDBConnection); property errors : String read FErrors; + property CrawlerLog : TFslStringBuilder read FCrawlerLog write SetCrawlerLog; property OnSendEmail : TSendEmailEvent read FOnSendEmail write FOnSendEmail; class procedure test(db : TFDBManager); @@ -186,10 +195,26 @@ constructor TPackageUpdater.Create(zulip: TZulipTracker); destructor TPackageUpdater.Destroy; begin + FCrawlerLog.free; FZulip.free; inherited; end; +procedure TPackageUpdater.clog(s: String; mode: TCrawlerLogMode); +begin + case mode of + clmStart: FCrawlerLog.append('

'+FormatTextToHTML(s)+'

'#13#10); + clmHeader: FCrawlerLog.append('

'+FormatTextToHTML(s)+'

'#13#10); + clmError: + begin + FCrawlerLog.append('
  • '+FormatTextToHTML(s)+'
  • '#13#10); + FHasErrors := true; + end; + clmWarning: FCrawlerLog.append('
  • '+FormatTextToHTML(s)+'
  • '#13#10); + clmNote: FCrawlerLog.append('
  • '+FormatTextToHTML(s)+'
  • '#13#10); + end; +end; + procedure TPackageUpdater.DoSendEmail(dest, subj, body: String); var dt : TDateTime; @@ -244,6 +269,12 @@ function TPackageUpdater.hasStored(guid: String): boolean; FDB.Terminate; end; +procedure TPackageUpdater.SetCrawlerLog(AValue: TFslStringBuilder); +begin + FCrawlerLog.free; + FCrawlerLog:=AValue; +end; + procedure TPackageUpdater.log(msg, source: String; error : boolean); begin if error then @@ -315,12 +346,19 @@ procedure TPackageUpdater.store(source, url, guid: String; date : TFslDateTime; id := npm.name; version := npm.version; if (id+'#'+version <> idver) then + begin log('Warning processing '+idver+': actually found '+id+'#'+version+' in the package', source, true); + clog(idver+': actually found '+id+'#'+version+' in the package', clmWarning); + end; + description := npm.description; kind := npm.kind; canonical := npm.canonical; if npm.notForPublication then + begin log('Warning processing '+idver+': this package is not suitable for publication (likely broken links)', source, true); + clog(idver+': not suitable for publication (likely broken links)', clmWarning); + end; fhirVersion := npm.fhirVersion; if not isValidPackageId(id) then raise EPackageCrawlerException.Create('NPM Id "'+id+'" is not valid from '+source); @@ -329,6 +367,7 @@ procedure TPackageUpdater.store(source, url, guid: String; date : TFslDateTime; if (canonical = '') then begin log('Warning processing '+idver+': No canonical found in npm (from '+url+')', source, true); + clog(idver+': No canonical found in npm (from '+url+')', clmWarning); canonical := 'http://simplifier.net/packages/fictitious/'+id; end; if not isAbsoluteUrl(canonical) then @@ -381,6 +420,7 @@ procedure TPackageUpdater.update(DB : TFDBConnection); FDB := DB; try log('Fetch '+MASTER_URL, '', false); + clog('Master URL: '+MASTER_URL, clmStart); json := fetchJson(MASTER_URL); try pr := TPackageRestrictions.Create(json.arr['package-restrictions'].Link); @@ -394,6 +434,7 @@ procedure TPackageUpdater.update(DB : TFDBConnection); finally json.free; end; + clog('', clmHeader); except on e : EAbort do begin @@ -443,8 +484,11 @@ procedure TPackageUpdater.updateTheFeed(url, source, email: String; pr : TPackag if Logging.shuttingDown then Abort; try + clog('Process '+url, clmHeader); + FCrawlerLog.append('
      '#13#10); log('Fetch '+url, source, false); FFeedErrors := ''; + FHasErrors := false; xml := fetchXml(url); try @@ -466,11 +510,16 @@ procedure TPackageUpdater.updateTheFeed(url, source, email: String; pr : TPackag finally xml.free; end; + if not FHasErrors then + clog('All OK', clmNote); if (FFeedErrors <> '') and (email <> '') then DoSendEmail(email, 'Errors Processing '+url, FFeedErrors); + FCrawlerLog.append('
    '#13#10); except on e : Exception do - begin + begin + clog('Exception: '+e.Message, clmError); + FCrawlerLog.append(''#13#10); log('Exception processing feed: '+url+': '+e.Message, source, false); if (email <> '') then DoSendEmail(email, 'Exception Processing '+url, e.Message); @@ -487,39 +536,58 @@ procedure TPackageUpdater.updateItem(source : String; item: TMXmlElement; i : in begin if Logging.shuttingDown then Abort; - url := '??pck'; + url := '[link not found]'; if item.element('guid') = nil then begin log('Error processing item from '+source+'#item['+inttostr(i)+']: no guid provided', source, true); + clog('item['+inttostr(i)+']: no guid provided', clmError); exit; end; - if (item.element('notForPublication') <> nil) and ('true' = item.element('notForPublication').text) then - exit; guid := item.element('guid').Text; try id := item.element('title').Text; + if (item.element('notForPublication') <> nil) and ('true' = item.element('notForPublication').text) then + begin + clog(guid+': not for publication', clmError); + exit; + end; if pr.isOk(id, source, list) then begin if (not hasStored(guid)) then begin - d := item.element('pubDate').Text.Replace(' ', ' ').Substring(5); + d := item.element('pubDate').Text.toLower.Replace(' ', ' '); + if (d.substring(0, 6).contains(',')) then + d := d.substring(d.indexOf(',')+1) + else if StringStartsWith(d, ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']) then + d := d.substring(d.indexOf(' ')+1); + d := d.trim; if (d.length > 2) and (d[2] = ' ') and StringIsInteger16(d[1]) then d := '0'+d; - date := TFslDateTime.fromFormat('dd mmm yyyy hh:nn:ss', d); + try + date := TFslDateTime.fromFormat('dd mmm yyyy hh:nn:ss', d); + except + date := TFslDateTime.fromFormat('dd mmmm yyyy hh:nn:ss', d); + end; url := fix(item.element('link').Text); log('Fetch '+url, source, false); content := fetchUrl(url, 'application/tar+gzip'); store(source, url, guid, date, content, id); + clog(guid+': Fetched '+url, clmNote); end; end else begin - log('The package '+id+' is not allowed to come from '+source+' (allowed: '+list+')', source, true); + if not (source.contains('simplifier.net')) then + begin + log('The package '+id+' is not allowed to come from '+source+' (allowed: '+list+')', source, true); + clog(guid+': The package '+id+' is not allowed to come from '+source+' (allowed: '+list+')', clmError); + end; end; except on e : Exception do begin log('Exception processing item: '+guid+' from '+url+': '+e.Message, source, true); + clog(guid+': '+e.Message, clmError); end; end; end; diff --git a/server/web/packages-home.html b/server/web/packages-home.html index 8c11408bc..dcf4bc162 100644 --- a/server/web/packages-home.html +++ b/server/web/packages-home.html @@ -33,6 +33,9 @@

    FHIR Package Server

    [%count%] Packages available. [%downloads%] Packages Downloaded. Status = [%status%]. Package History last 30 days

    +

    +Last Package Crawler Log +

    How to add packages to this package server: diff --git a/server/web/packages-log.html b/server/web/packages-log.html new file mode 100644 index 000000000..06b100012 --- /dev/null +++ b/server/web/packages-log.html @@ -0,0 +1,19 @@ + + + + + Package Server (FHIR Server [%id%] Version [%ver%]) + [%include head.html%] + + + +[%include top.html%] +

    FHIR Package Server - Crawler Log

    + +[%log%] + +[%include bottom.html%] + + + + diff --git a/server/web/packages-search.html b/server/web/packages-search.html index 1d05782a4..0af85515b 100644 --- a/server/web/packages-search.html +++ b/server/web/packages-search.html @@ -9,6 +9,11 @@ [%include top.html%]

    FHIR Package Server

    + +

    + Last Package Crawler Log +

    +

    Search the Catalog:

    diff --git a/toolkit2/fhirtoolkit.lpi b/toolkit2/fhirtoolkit.lpi index b6dbcc5a4..c3dc1ba3f 100644 --- a/toolkit2/fhirtoolkit.lpi +++ b/toolkit2/fhirtoolkit.lpi @@ -17,7 +17,7 @@ - +