rewrite lua module updater, now working with directories

-github api has a limitation of 5000 queries per day. so authorization token is needed here.
-added simple dialog with tmemo. maybe need to fix later
-todo: move non website module file to other director-y

baseunits/FMDOptions.pas

@@ -97,7 +97,8 @@   TIniFileRun = class(IniFiles.TMemIniFile)
   EXTRAS_FOLDER,
   MANGAFOXTEMPLATE_FOLDER,
   LUA_WEBSITEMODULE_FOLDER,
-  LUA_WEBSITEMODULE_FILE,
+  LUA_REPO_FOLDER,
+  LUA_REPO_FILE,
   BACKUP_FOLDER: String;
 
   // dll
@@ -330,7 +331,7 @@ procedure SetAppDataDirectory(const ADir: String);
   CONFIG_FILE := CONFIG_FOLDER + 'config.ini';
   ACCOUNTS_FILE := CONFIG_FOLDER + 'accounts.db';
   MODULES_FILE := CONFIG_FOLDER + 'modules.json';
-  LUA_WEBSITEMODULE_FILE := CONFIG_FOLDER + 'luamodules.json';
+  LUA_REPO_FILE := CONFIG_FOLDER + 'lua.json';
 
   DATA_FOLDER := APPDATA_DIRECTORY + 'data' + PathDelim;
 
@@ -343,6 +344,7 @@ procedure SetAppDataDirectory(const ADir: String);
   FAVORITESDB_FILE := WORK_FOLDER + 'favorites.db';
 
   LUA_WEBSITEMODULE_FOLDER := FMD_DIRECTORY + 'lua' + PathDelim + 'modules' + PathDelim;
+  LUA_REPO_FOLDER := FMD_DIRECTORY + 'lua' + PathDelim;
 
   SetIniFiles;
 end;

baseunits/GithubRepo.pas

@@ -0,0 +1,271 @@
+unit GitHubRepo;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  Classes, SysUtils, httpsendthread, BaseThread, fpjson;
+
+type
+{ TGitHubRepo }
+
+  TGitHubRepo = class
+  private
+    ConfigFile: String;
+    HTTP: THTTPSendThread;
+    Tree: TJSONArray;
+    Props: TJSONObject;
+  protected
+    procedure SetAuth;
+    function QueryLastCommit: String;
+    function QueryTree: String;
+    function QueryProps(const ANames: TStrings): String;
+  public
+    api_url,
+    download_url,
+    owner,
+    name,
+    token,
+    ref,
+    path,
+    last_commit: String;
+    max_deep: Integer;
+    constructor Create(const AConfigFile: String; const AThread: TBaseThread = nil);
+    destructor Destroy; override;
+    function GetLastCommit: String;
+    function GetTree: TJSONArray;
+    function GetDownloadURL(const AName: String): String;
+    function GetProps(const ANames: TStrings): TJSONObject;
+  end;
+
+implementation
+
+uses jsonparser, jsonscanner, IniFiles, uBaseUnit;
+
+{ TGitHubRepo }
+
+procedure TGitHubRepo.SetAuth;
+begin
+  HTTP.Headers.Values['Authorization']:=' bearer '+token;
+end;
+
+function TGitHubRepo.QueryLastCommit: String;
+var
+  lpath: String;
+begin
+  lpath:=path; if lpath<>'' then lpath := ', path: "'+lpath+'"';
+  Result := '{"query": "'+StringToJSONString(
+               '{'+
+                 'repository(owner: "'+owner+'", name: "'+name+'") {'+
+                   'ref(qualifiedName: "'+ref+'") {'+
+                     'target {'+
+                       '... on Commit {'+
+                         'history(first: 1'+lpath+'){ '+
+                           'nodes {'+
+                             'oid'+
+                           '}'+
+                         '}'+
+                       '}'+
+                     '}'+
+                   '}'+
+                 '}'+
+               '}'
+         )+'"}';
+end;
+
+function TGitHubRepo.QueryTree: String;
+
+  function onTree(const x: Integer):string;
+  begin
+    Result:= '... on Tree {'+
+                'entries {'+
+                  'oid '+
+                  'name ';
+    if x>1 then
+      Result+=    'object {'+
+                    onTree(x-1)+
+                  '}';
+    Result+=    '}'+
+             '}';
+  end;
+begin
+  Result := '{"query": "'+StringToJSONString(
+              '{'+
+                'repository(owner: "'+owner+'", name: "'+name+'") {'+
+                  'object(expression: "'+last_commit+':'+path+'") {'+
+                    onTree(max_deep)+
+                  '}'+
+                '}'+
+              '}'
+         )+'"}';
+end;
+
+function TGitHubRepo.QueryProps(const ANames: TStrings): String;
+  function onProps:String;
+  var
+    i: Integer;
+    lpath: String;
+  begin
+    Result:='';
+    lpath :=path; if lpath<>'' then lpath+='/';
+    for i:=0 to ANames.Count-1 do
+    begin
+      Result += 'p'+IntToStr(i)+': history(path: "'+lpath+ANames[i]+'", first: 1) {'+
+                  'nodes {'+
+                    'message '+
+                    'committedDate'+
+                  '}'+
+                '} ';
+    end;
+  end;
+begin
+  Result := '{"query": "'+StringToJSONString(
+              '{'+
+                'repository(owner: "'+owner+'", name: "'+name+'") {'+
+                  'object(oid: "'+last_commit+'") {'+
+                    '... on Commit {'+
+                      onProps+
+                    '}'+
+                  '}'+
+                '}'+
+              '}'
+         )+'"}';
+end;
+
+constructor TGitHubRepo.Create(const AConfigFile: String;
+  const AThread: TBaseThread);
+begin
+  ConfigFile := AConfigFile;
+  HTTP := THTTPSendThread.Create(AThread);
+  if FileExists(ConfigFile) then
+    with TIniFile.Create(ConfigFile) do
+      try
+        api_url      := ReadString               ('GitHub', 'api_url'     , '');
+        download_url := ReadString               ('GitHub', 'download_url', '');
+        owner        := ReadString               ('GitHub', 'owner'       , '');
+        name         := ReadString               ('GitHub', 'name'        , '');
+        token        := DecryptString(ReadString ('GitHub', 'token'       , ''));
+        ref          := ReadString               ('GitHub', 'ref'         , '');
+        path         := ReadString               ('GitHub', 'path'        , '');
+        max_deep     := ReadInteger              ('GitHub', 'max_deep'    , 2);
+      finally
+        Free;
+      end;
+  if api_url  = '' then api_url  := 'https://api.github.com/graphql';
+  if ref      = '' then ref      := 'master';
+  if max_deep < 1  then max_deep := 1;
+  last_commit:= ref;
+end;
+
+destructor TGitHubRepo.Destroy;
+begin
+  if Assigned(Tree) then Tree.Free;
+  if Assigned(Props) then Props.Free;
+  HTTP.Free;
+  inherited Destroy;
+end;
+
+function TGitHubRepo.GetLastCommit: String;
+var
+  d: TJSONData;
+  a: TJSONArray;
+begin
+  last_commit:='';
+  HTTP.Reset;
+  SetAuth;
+  d:=nil;
+  if HTTP.POST(api_url, QueryLastCommit) then
+    with TJSONParser.Create(HTTP.Document, [joUTF8]) do
+      try
+        d:=Parse;
+      finally
+        free;
+      end;
+  if Assigned(d) then
+    try
+      a:=TJSONArray(d.GetPath('data.repository.ref.target.history.nodes'));
+      if Assigned(a) then
+        last_commit:=TJSONObject(a.Items[0]).Get('oid','');
+    except
+    end;
+    d.free;
+  Result := last_commit;
+end;
+
+function TGitHubRepo.GetTree: TJSONArray;
+var
+  d: TJSONData;
+  a: TJSONArray;
+begin
+  result:=nil;
+  HTTP.Reset;
+  SetAuth;
+  if last_commit='' then
+    last_commit := 'master';
+  d:=nil;
+  if HTTP.POST(api_url, QueryTree) then
+    with TJSONParser.Create(HTTP.Document, [joUTF8]) do
+      try
+        d:=Parse;
+      finally
+        free;
+      end;
+  if Assigned(Tree) then
+    FreeAndNil(Tree);
+  if Assigned(d) then
+  begin
+    try
+      a:=TJSONArray(d.GetPath('data.repository.object.entries'));
+      if Assigned(a) then
+        Tree:=TJSONArray(a.Clone);
+    except
+    end;
+    d.free;
+  end;
+  Result := Tree;
+end;
+
+function TGitHubRepo.GetDownloadURL(const AName: String): String;
+var
+  lpath: String;
+begin
+  lpath:=path; if lpath<>'' then lpath:=lpath+'/';
+  Result:=AppendURLDelim(download_url)+owner+'/'+name+'/'+ref+'/'+lpath+AName;
+end;
+
+function TGitHubRepo.GetProps(const ANames: TStrings): TJSONObject;
+var
+  d: TJSONData;
+  o: TJSONObject;
+begin
+  result:=nil;
+  HTTP.Reset;
+  SetAuth;
+  if last_commit='' then
+    last_commit := 'master';
+  d:=nil;
+  if HTTP.POST(api_url, QueryProps(ANames)) then
+    with TJSONParser.Create(HTTP.Document, [joUTF8]) do
+      try
+        d:=Parse;
+      finally
+        free;
+      end;
+  if Assigned(Props) then
+    FreeAndNil(Props);
+  if Assigned(d) then
+  begin
+    try
+      o:=TJSONObject(d.GetPath('data.repository.object'));
+      if Assigned(o) then
+        Props:=TJSONObject(o.Clone);
+    except
+    end;
+    d.free;
+  end;
+  Result := Props;
+end;
+
+end.
+

changelog.txt

@@ -4,6 +4,10 @@ https://github.com/fmd-project-team/FMD
 Changelog:
 ([!] Important, [+] Addition, [-] Removal, [*] Fix/Change, [?] Feedback needed)
 
+1.2.0.0 (22.02.2020)
+[*] Lua modules structure changed
+Full changes: https://github.com/fmd-project-team/FMD/compare/1.1.4.0...1.2.0.0
+
 1.1.4.0 (15.02.2020)
 [*] Various changes and bug fixes
 Full changes: https://github.com/fmd-project-team/FMD/compare/1.1.3.0...1.1.4.0

config/base.ini

@@ -4,5 +4,13 @@ DB_URL=https://raw.githubusercontent.com/fmd-project-team/FMD-WebsiteDatabases/m
 UPDATE_URL=https://raw.githubusercontent.com/fmd-project-team/FMD/master/update
 CHANGELOG_URL=https://raw.githubusercontent.com/fmd-project-team/FMD/master/changelog.txt
 UPDATE_PACKAGE_NAME=updatepackage.7z
-MODULES_URL=https://api.github.com/repos/fmd-project-team/FMD/contents/lua/modules
-MODULES_URL2=https://github.com/fmd-project-team/FMD/file-list/master/lua/modules
+
+[GitHub]
+api_url=https://api.github.com/graphql
+download_url=https://raw.githubusercontent.com/
+owner=fmd-project-team
+name=FMD
+token=q0NI9Ztgiv1z5s5sYwscdz/CCYQnfGkbRbDGd02dwhsFzXNH9WPNWg==
+ref=master
+path=lua
+max_deep=4

make_release_win.bat

@@ -8,10 +8,10 @@ SET repodl=https://github.com/fmd-project-team/FMD/releases/download/
 ECHO ; automatically build with make_release_win.bat>update
 
 CALL :makerelease i386-win32 Win32 --no-write-project
-ECHO WIN32=%repodl%%fver%/%oname%>>update
+ECHO WIN32=%repodl%%fverb%/%oname%>>update
 
 CALL :makerelease x86_64-win64 Win64 --no-write-project
-ECHO WIN64=%repodl%%fver%/%oname%>>update
+ECHO WIN64=%repodl%%fverb%/%oname%>>update
 
 ECHO VERSION=%fverb%>>update