Consume new ghtutorial route when loading tutorials from github (#10217)

* consume new ghtutorial route when loading tutorials from github

* add support for docs tutorial route and use etags

* revert testing change

* add iframe message for caching tutorial code

* always update the cache

* add editor extension for loading markdown activities

cli/cli.ts

@@ -252,6 +252,14 @@ class FileGithubDb implements pxt.github.IGithubDb {
     loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
         return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
     }
+
+    loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {
+        return this.loadAsync(repopath, tag, "tutorial", (r, t) => this.db.loadTutorialMarkdown(r, t));
+    }
+
+    cacheReposAsync(resp: pxt.github.GHTutorialResponse) {
+        return this.db.cacheReposAsync(resp);
+    }
 }
 
 function searchAsync(...query: string[]) {

localtypings/pxteditor.d.ts

@@ -102,6 +102,7 @@ declare namespace pxt.editor {
         | "editorcontentloaded"
         | "serviceworkerregistered"
         | "runeval"
+        | "precachetutorial"
 
         // package extension messasges
         | ExtInitializeType
@@ -408,6 +409,12 @@ declare namespace pxt.editor {
         carryoverPreviousCode?: boolean;
     }
 
+    export interface PrecacheTutorialRequest extends EditorMessageRequest {
+        action: "precachetutorial";
+        data: pxt.github.GHTutorialResponse;
+        lang?: string;
+    }
+
     export interface InfoMessage {
         versions: pxt.TargetVersions;
         locale: string;
@@ -1140,6 +1147,7 @@ declare namespace pxt.editor {
 
         // Used with the @tutorialCompleted macro. See docs/writing-docs/tutorials.md for more info
         onTutorialCompleted?: () => void;
+        onMarkdownActivityLoad?: (path: string, title?: string, editorProjectName?: string) => Promise<void>;
 
         // Used with @codeStart, @codeStop metadata (MINECRAFT HOC ONLY)
         onCodeStart?: () => void;

pxtcompiler/simpledriver.ts

@@ -55,6 +55,14 @@ namespace pxt {
         loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
             return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
         }
+
+        loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {
+            return this.loadAsync(repopath, tag, "tutorial", (r, t) => this.db.loadTutorialMarkdown(r, t));
+        }
+
+        cacheReposAsync(resp: pxt.github.GHTutorialResponse) {
+            return this.db.cacheReposAsync(resp);
+        }
     }
 
     function pkgOverrideAsync(pkg: pxt.Package) {

pxteditor/editorcontroller.ts

@@ -170,7 +170,7 @@ export function bindEditorMessages(getEditorAsync: () => Promise<IProjectView>)
                                         })
                                     });
                             }
-case "renderxml": {
+                            case "renderxml": {
                                 const rendermsg = data as pxt.editor.EditorMessageRenderXmlRequest;
                                 return Promise.resolve()
                                     .then(() => {
@@ -303,6 +303,20 @@ case "renderxml": {
                                 }
                                 return projectView.setLanguageRestrictionAsync(msg.restriction);
                             }
+                            case "precachetutorial": {
+                                const msg = data as pxt.editor.PrecacheTutorialRequest;
+                                const tutorialData = msg.data;
+                                const lang = msg.lang || pxt.Util.userLanguage();
+
+                                return pxt.github.db.cacheReposAsync(tutorialData)
+                                    .then(async () => {
+                                        if (typeof tutorialData.markdown === "string") {
+                                            // the markdown needs to be cached in the translation db
+                                            const db = await pxt.BrowserUtils.translationDbAsync();
+                                            await db.setAsync(lang, tutorialData.path, undefined, undefined, tutorialData.markdown);
+                                        }
+                                    });
+                            }
                         }
                         return Promise.resolve();
                     });

pxtlib/browserutils.ts

@@ -1409,6 +1409,24 @@ namespace pxt.BrowserUtils {
         return `${url}${url.indexOf('?') > 0 ? "&" : "?"}rnd=${Math.random()}`
     }
 
+    export function appendUrlQueryParams(url: string, params: URLSearchParams) {
+        const entries: string[] = [];
+        for (const [key, value] of params.entries()) {
+            entries.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`);
+        }
+
+        if (entries.length) {
+            if (url.indexOf("?") !== -1) {
+                url += "&" + entries.join("&");
+            }
+            else {
+                url += "?" + entries.join("&");
+            }
+        }
+
+        return url;
+    }
+
     export function legacyCopyText(element: HTMLInputElement | HTMLTextAreaElement) {
         element.focus();
         element.setSelectionRange(0, 9999);

pxtlib/cmds.ts

@@ -39,4 +39,5 @@ namespace pxt.commands {
     export let webUsbPairDialogAsync: (pairAsync: () => Promise<boolean>, confirmAsync: (options: any) => Promise<WebUSBPairResult>, implicitlyCalled?: boolean) => Promise<WebUSBPairResult> = undefined;
     export let onTutorialCompleted: () => void = undefined;
     export let workspaceLoadedAsync: () => Promise<void> = undefined;
+    export let onMarkdownActivityLoad: (path: string, title?: string, editorProjectName?: string) => Promise<void> = undefined;
 }

pxtlib/emitter/cloud.ts

@@ -32,6 +32,7 @@ namespace pxt.Cloud {
     export function useCdnApi() {
         return pxt.webConfig && !pxt.webConfig.isStatic
             && !BrowserUtils.isLocalHost() && !!pxt.webConfig.cdnUrl
+            && !/nocdn=1/i.test(window.location.href);
     }
 
     export function cdnApiUrl(url: string) {
@@ -135,7 +136,7 @@ namespace pxt.Cloud {
         return resp.json;
     }
 
-    export async function markdownAsync(docid: string, locale?: string, propagateExceptions?: boolean): Promise<string> {
+    export async function markdownAsync(docid: string, locale?: string, propagateExceptions?: boolean, downloadTutorialBundle?: boolean): Promise<string> {
         // 1h check on markdown content if not on development server
         const MARKDOWN_EXPIRATION = pxt.BrowserUtils.isLocalHostDev() ? 0 : 1 * 60 * 60 * 1000;
         // 1w check don't use cached version and wait for new content
@@ -148,7 +149,7 @@ namespace pxt.Cloud {
 
         const downloadAndSetMarkdownAsync = async () => {
             try {
-                const r = await downloadMarkdownAsync(docid, locale, entry?.etag);
+                const r = await downloadMarkdownAsync(docid, locale, entry?.etag, downloadTutorialBundle);
                 // TODO directly compare the entry/response etags after backend change
                 if (!entry || (r.md && entry.md !== r.md)) {
                     await db.setAsync(locale, docid, r.etag, undefined, r.md);
@@ -189,11 +190,13 @@ namespace pxt.Cloud {
         return downloadAndSetMarkdownAsync();
     }
 
-    function downloadMarkdownAsync(docid: string, locale?: string, etag?: string): Promise<{ md: string; etag?: string; }> {
+    async function downloadMarkdownAsync(docid: string, locale?: string, etag?: string, downloadTutorialBundle?: boolean): Promise<{ md: string; etag?: string; }> {
         const packaged = pxt.webConfig?.isStatic;
         const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target || '?';
         let url: string;
 
+        const searchParams = new URLSearchParams();
+
         if (packaged) {
             url = docid;
             const isUnderDocs = /\/?docs\//.test(url);
@@ -205,24 +208,51 @@ namespace pxt.Cloud {
             if (!hasExt) {
                 url = `${url}.md`;
             }
-        } else {
-            url = `md/${pxt.appTarget.id}/${docid.replace(/^\//, "")}?targetVersion=${encodeURIComponent(targetVersion)}`;
+        }
+        else {
+            url = `md/${pxt.appTarget.id}/${docid.replace(/^\//, "")}`;
+            searchParams.set("targetVersion", targetVersion);
         }
         if (locale != "en") {
-            url += `${packaged ? "?" : "&"}lang=${encodeURIComponent(locale)}`
+            searchParams.set("lang", locale);
         }
+
+        url = pxt.BrowserUtils.appendUrlQueryParams(url, searchParams);
+
         if (pxt.BrowserUtils.isLocalHost() && !pxt.Util.liveLocalizationEnabled()) {
             return localRequestAsync(url).then(resp => {
                 if (resp.statusCode == 404)
                     return privateRequestAsync({ url, method: "GET" })
                         .then(resp => { return { md: resp.text, etag: <string>resp.headers["etag"] }; });
                 else return { md: resp.text, etag: undefined };
             });
-        } else {
-            const headers: pxt.Map<string> = etag && !useCdnApi() ? { "If-None-Match": etag } : undefined;
-            return apiRequestWithCdnAsync({ url, method: "GET", headers })
-                .then(resp => { return { md: resp.text, etag: <string>resp.headers["etag"] }; });
         }
+
+        const headers: pxt.Map<string> = etag && !useCdnApi() ? { "If-None-Match": etag } : undefined;
+
+        let resp: Util.HttpResponse;
+        let md: string;
+
+        if (!packaged && downloadTutorialBundle) {
+            url = url.replace(/^md\//, "ghtutorial/docs/");
+            resp = await apiRequestWithCdnAsync({ url, method: "GET", headers });
+
+            const body = resp.json as pxt.github.GHTutorialResponse;
+            await pxt.github.db.cacheReposAsync(body);
+
+            md = body.markdown as string;
+        }
+        else {
+            resp = await apiRequestWithCdnAsync({ url, method: "GET", headers });
+            md = resp.text;
+        }
+
+        return (
+            {
+                md,
+                etag: (resp.headers["etag"] as string)
+            }
+        );
     }
 
     export function privateDeleteAsync(path: string) {

pxtlib/github.ts

@@ -75,6 +75,23 @@ namespace pxt.github {
         user: User;
     }
 
+    export interface GHTutorialResponse {
+        path: string;
+        markdown: string | { filename: string, repo: GHTutorialRepoInfo };
+        dependencies: GHTutorialRepoInfo[];
+    }
+
+    export interface GHTutorialRepoInfo {
+        repo: string;
+        files: pxt.Map<string>;
+        sha: string;
+        fileHash: string;
+
+        subPath?: string;
+        version?: string;
+        latestVersion?: string;
+    }
+
     export let forceProxy = false;
 
     function hasProxy() {
@@ -108,6 +125,8 @@ namespace pxt.github {
         latestVersionAsync(repopath: string, config: PackagesConfig): Promise<string>;
         loadConfigAsync(repopath: string, tag: string): Promise<pxt.PackageConfig>;
         loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage>;
+        loadTutorialMarkdown(repopath: string, tag?: string): Promise<CachedPackage>;
+        cacheReposAsync(response: GHTutorialResponse): Promise<void>;
     }
 
     function ghRequestAsync(options: U.HttpRequestOptions) {
@@ -279,6 +298,58 @@ namespace pxt.github {
                         })
                 })
         }
+
+        async loadTutorialMarkdown(repoPath: string, tag?: string) {
+            const tutorialResponse = (await downloadMarkdownTutorialInfoAsync(repoPath, tag)).resp;
+
+            const repo = tutorialResponse.markdown as { filename: string, repo: GHTutorialRepoInfo };
+
+            pxt.Util.assert(typeof repo === "object");
+
+            await this.cacheReposAsync(tutorialResponse);
+
+            return repo.repo;
+        }
+
+        async cacheReposAsync(resp: GHTutorialResponse) {
+            if (typeof resp.markdown === "object") {
+                const repo = resp.markdown as { filename: string, repo: GHTutorialRepoInfo };
+
+                this.cacheRepo(repo.repo);
+            }
+            for (const dep of resp.dependencies) {
+                this.cacheRepo(dep);
+            }
+        }
+
+        private cacheRepo(repo: GHTutorialRepoInfo) {
+            let repopath = repo.repo;
+
+            if (repo.subPath) {
+                repopath += "/" + repo.subPath;
+            }
+            let key = repopath
+            key += "/" + repo.sha;
+            this.packages[key] = {
+                files: repo.files
+            };
+
+            if (repo.latestVersion) {
+                this.cacheLatestVersion(repopath, repo.latestVersion);
+            }
+
+            const config = repo.files["pxt.json"];
+
+            if (config) {
+                const alternateConfigKey = key + "/" + (repo.version || "master");
+                this.cacheConfig(key, config);
+                this.cacheConfig(alternateConfigKey, config);
+            }
+        }
+
+        private cacheLatestVersion(repopath: string, version: string) {
+            this.latestVersions[repopath] = version;
+        }
     }
 
     function fallbackDownloadTextAsync(parsed: ParsedRepo, commitid: string, filepath: string) {
@@ -315,6 +386,51 @@ namespace pxt.github {
         })
     }
 
+    export async function downloadMarkdownTutorialInfoAsync(repopath: string, tag?: string, noCache?: boolean, etag?: string): Promise<{ resp?: GHTutorialResponse, etag?: string }> {
+        let request = pxt.Cloud.apiRequestWithCdnAsync;
+        const queryParams = new URLSearchParams();
+        if (tag) {
+            queryParams.set("ref", tag);
+        }
+        if (noCache) {
+            queryParams.set("noCache", "1");
+            request = pxt.Cloud.privateRequestAsync;
+        }
+
+        let url = `ghtutorial/${repopath}`;
+        url = pxt.BrowserUtils.appendUrlQueryParams(url, queryParams);
+
+        const headers: pxt.Map<string> = etag ? { "If-None-Match": etag } : undefined;
+
+        const resp = await request(
+            {
+                url,
+                method: "GET",
+                headers
+            }
+        );
+
+        let body: GHTutorialResponse;
+
+        if (resp.statusCode === 304) {
+            body = undefined;
+        }
+        else {
+            body = resp.json;
+        }
+
+        return (
+            {
+                resp: body,
+                etag: resp.headers["etag"] as string
+            }
+        );
+    }
+
+    export async function downloadTutorialMarkdownAsync(repopath: string, tag?: string) {
+        return db.loadTutorialMarkdown(repopath, tag);
+    }
+
     // overriden by client
     export let db: IGithubDb = new MemoryGithubDb();
 

pxtlib/tutorial.ts

@@ -505,7 +505,11 @@ ${code}
 
     export function resolveLocalizedMarkdown(ghid: pxt.github.ParsedRepo, files: pxt.Map<string>, fileName?: string): string {
         // if non-default language, find localized file if any
-        const mfn = (fileName || ghid.fileName || "README") + ".md";
+        let mfn = (fileName || ghid.fileName || "README");
+
+        if (!mfn.endsWith(".md")) {
+            mfn += ".md";
+        }
 
         let md: string = undefined;
         const [initialLang, baseLang, initialLangLowerCase] = pxt.Util.normalizeLanguageCode(pxt.Util.userLanguage());

pxtservices/editorDriver.ts

@@ -446,6 +446,16 @@ export class EditorDriver extends IframeDriver {
         );
     }
 
+    async precacheTutorial(data: pxt.github.GHTutorialResponse) {
+        await this.sendRequest(
+            {
+                type: "pxteditor",
+                action: "precachetutorial",
+                data
+            } as pxt.editor.PrecacheTutorialRequest
+        );
+    }
+
     addEventListener(event: typeof MessageSentEvent, handler: (ev: pxt.editor.EditorMessage) => void): void;
     addEventListener(event: typeof MessageReceivedEvent, handler: (ev: pxt.editor.EditorMessage) => void): void;
     addEventListener(event: "event", handler: (ev: pxt.editor.EditorMessageEventRequest) => void): void;