Merge branch 'main' into index-default-branch

Author: Steven-Nagie

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Fixed issue where we crash on startup if the install / upgrade PostHog event fails to send. ([#159](https://github.com/sourcebot-dev/sourcebot/pull/159))
+- Fixed issue with broken file links. ([#161](https://github.com/sourcebot-dev/sourcebot/pull/161))
+
+## [2.7.0] - 2025-01-10
+
+### Added
+
+- Added support for creating share links to snippets of code. ([#149](https://github.com/sourcebot-dev/sourcebot/pull/149))
+- Added support for indexing raw remote git repository. ([#152](https://github.com/sourcebot-dev/sourcebot/pull/152))
+
 ## [2.6.3] - 2024-12-18
 
 ### Added

Dockerfile

@@ -1,5 +1,5 @@
 FROM node:20-alpine3.19 AS node-alpine
-FROM golang:1.22.2-alpine3.19 AS go-alpine
+FROM golang:1.23.4-alpine3.19 AS go-alpine
 
 # ------ Build Zoekt ------
 FROM go-alpine AS zoekt-builder

entrypoint.sh

@@ -30,14 +30,16 @@ if [ ! -f "$FIRST_RUN_FILE" ]; then
     # If this is our first run, send a `install` event to PostHog
     # (if telemetry is enabled)
     if [ -z "$SOURCEBOT_TELEMETRY_DISABLED" ]; then
-        curl -L -s --header "Content-Type: application/json" -d '{
+        if ! ( curl -L --output /dev/null --silent --fail --header "Content-Type: application/json" -d '{
             "api_key": "'"$POSTHOG_PAPIK"'",
             "event": "install",
             "distinct_id": "'"$SOURCEBOT_INSTALL_ID"'",
             "properties": {
                 "sourcebot_version": "'"$SOURCEBOT_VERSION"'"
             }
-        }' https://us.i.posthog.com/capture/ > /dev/null
+        }' https://us.i.posthog.com/capture/ ) then
+            echo -e "\e[33m[Warning] Failed to send install event.\e[0m"
+        fi
     fi
 else
     export SOURCEBOT_INSTALL_ID=$(cat "$FIRST_RUN_FILE" | jq -r '.install_id')
@@ -48,15 +50,17 @@ else
         echo -e "\e[34m[Info] Upgraded from version $PREVIOUS_VERSION to $SOURCEBOT_VERSION\e[0m"
 
         if [ -z "$SOURCEBOT_TELEMETRY_DISABLED" ]; then
-            curl -L -s --header "Content-Type: application/json" -d '{
+            if ! ( curl -L --output /dev/null --silent --fail --header "Content-Type: application/json" -d '{
                 "api_key": "'"$POSTHOG_PAPIK"'",
                 "event": "upgrade",
                 "distinct_id": "'"$SOURCEBOT_INSTALL_ID"'",
                 "properties": {
                     "from_version": "'"$PREVIOUS_VERSION"'",
                     "to_version": "'"$SOURCEBOT_VERSION"'"
                 }
-            }' https://us.i.posthog.com/capture/ > /dev/null
+            }' https://us.i.posthog.com/capture/ ) then
+                echo -e "\e[33m[Warning] Failed to send upgrade event.\e[0m"
+            fi
         fi
     fi
 fi

packages/backend/src/git.ts

@@ -1,7 +1,9 @@
-import { GitRepository } from './types.js';
+import { GitRepository, AppContext } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { existsSync } from 'fs';
 import { createLogger } from './logger.js';
+import { GitConfig } from './schemas/v2.js';
+import path from 'path';
 
 const logger = createLogger('git');
 
@@ -48,4 +50,81 @@ export const fetchRepository = async (repo: GitRepository, onProgress?: (event:
             "--progress"
         ]
     );
+}
+
+const isValidGitRepo = async (url: string): Promise<boolean> => {
+    const git = simpleGit();
+    try {
+        await git.listRemote([url]);
+        return true;
+    } catch (error) {
+        logger.debug(`Error checking if ${url} is a valid git repo: ${error}`);
+        return false;
+    }
+}
+
+const stripProtocolAndGitSuffix = (url: string): string => {
+    return url.replace(/^[a-zA-Z]+:\/\//, '').replace(/\.git$/, '');
+}
+
+const getRepoNameFromUrl = (url: string): string => {
+    const strippedUrl = stripProtocolAndGitSuffix(url);
+    return strippedUrl.split('/').slice(-2).join('/');
+}
+
+export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) => {
+    const repoValid = await isValidGitRepo(config.url);
+    if (!repoValid) {
+        logger.error(`Git repo provided in config with url ${config.url} is not valid`);
+        return null;
+    }
+
+    const cloneUrl = config.url;
+    const repoId = stripProtocolAndGitSuffix(cloneUrl);
+    const repoName = getRepoNameFromUrl(config.url);
+    const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
+    const repo: GitRepository = {
+        vcs: 'git',
+        id: repoId,
+        name: repoName,
+        path: repoPath,
+        isStale: false,
+        cloneUrl: cloneUrl,
+        branches: [],
+        tags: [],
+    }
+
+    if (config.revisions) {
+        if (config.revisions.branches) {
+            const branchGlobs = config.revisions.branches;
+            const git = simpleGit();
+            const branchList = await git.listRemote(['--heads', cloneUrl]);
+            const branches = branchList
+                .split('\n')
+                .map(line => line.split('\t')[1])
+                .filter(Boolean)
+                .map(branch => branch.replace('refs/heads/', ''));
+
+            repo.branches = branches.filter(branch => 
+                branchGlobs.some(glob => new RegExp(glob).test(branch))
+            );
+        }
+
+        if (config.revisions.tags) {
+            const tagGlobs = config.revisions.tags;
+            const git = simpleGit();
+            const tagList = await git.listRemote(['--tags', cloneUrl]);
+            const tags = tagList
+                .split('\n')
+                .map(line => line.split('\t')[1])
+                .filter(Boolean)
+                .map(tag => tag.replace('refs/tags/', ''));
+
+            repo.tags = tags.filter(tag => 
+                tagGlobs.some(glob => new RegExp(glob).test(tag))
+            );
+        }
+    }
+
+    return repo;
 }

packages/backend/src/main.ts

@@ -6,7 +6,7 @@ import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
 import { getGerritReposFromConfig } from "./gerrit.js";
 import { AppContext, LocalRepository, GitRepository, Repository, Settings } from "./types.js";
-import { cloneRepository, fetchRepository } from "./git.js";
+import { cloneRepository, fetchRepository, getGitRepoFromConfig } from "./git.js";
 import { createLogger } from "./logger.js";
 import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js';
 import { arraysEqualShallow, stringsEqualFalseySafe, isRemotePath, measure } from "./utils.js";
@@ -246,6 +246,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(repo);
                 break;
             }
+            case 'git': {
+                const gitRepo = await  getGitRepoFromConfig(repoConfig, ctx);
+                gitRepo && configRepos.push(gitRepo);
+                break;
+            }
         }
     }
 

packages/backend/src/schemas/v2.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -268,3 +268,14 @@ export interface LocalConfig {
     paths?: string[];
   };
 }
+export interface GitConfig {
+  /**
+   * Git Configuration
+   */
+  type: "git";
+  /**
+   * The URL to the git repository.
+   */
+  url: string;
+  revisions?: GitRevisions;
+}

packages/web/package.json

@@ -35,6 +35,7 @@
     "@codemirror/search": "^6.5.6",
     "@codemirror/state": "^6.4.1",
     "@codemirror/view": "^6.33.0",
+    "@floating-ui/react": "^0.27.2",
     "@hookform/resolvers": "^3.9.0",
     "@iconify/react": "^5.1.0",
     "@iizukak/codemirror-lang-wgsl": "^0.3.0",
@@ -87,7 +88,7 @@
     "graphql": "^16.9.0",
     "http-status-codes": "^2.3.0",
     "lucide-react": "^0.435.0",
-    "next": "14.2.15",
+    "next": "14.2.21",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.161.5",
     "pretty-bytes": "^6.1.1",

packages/web/src/app/api/(client)/client.ts

@@ -3,6 +3,7 @@
 import { NEXT_PUBLIC_DOMAIN_SUB_PATH } from "@/lib/environment.client";
 import { fileSourceResponseSchema, listRepositoriesResponseSchema, searchResponseSchema } from "@/lib/schemas";
 import { FileSourceRequest, FileSourceResponse, ListRepositoriesResponse, SearchRequest, SearchResponse } from "@/lib/types";
+import assert from "assert";
 
 export const search = async (body: SearchRequest): Promise<SearchResponse> => {
     const path = resolveServerPath("/api/search");
@@ -48,5 +49,6 @@ export const getRepos = async (): Promise<ListRepositoriesResponse> => {
  * the base path (if any).
  */
 export const resolveServerPath = (path: string) => {
+    assert(path.startsWith("/"));
     return `${NEXT_PUBLIC_DOMAIN_SUB_PATH}${path}`;
 }

packages/web/src/app/browse/[...path]/codePreview.tsx

@@ -0,0 +1,150 @@
+'use client';
+
+import { ScrollArea } from "@/components/ui/scroll-area";
+import { useKeymapExtension } from "@/hooks/useKeymapExtension";
+import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam";
+import { useSyntaxHighlightingExtension } from "@/hooks/useSyntaxHighlightingExtension";
+import { useThemeNormalized } from "@/hooks/useThemeNormalized";
+import { search } from "@codemirror/search";
+import CodeMirror, { Decoration, DecorationSet, EditorSelection, EditorView, ReactCodeMirrorRef, SelectionRange, StateField, ViewUpdate } from "@uiw/react-codemirror";
+import { useEffect, useMemo, useRef, useState } from "react";
+import { EditorContextMenu } from "../../components/editorContextMenu";
+
+interface CodePreviewProps {
+    path: string;
+    repoName: string;
+    revisionName: string;
+    source: string;
+    language: string;
+}
+
+export const CodePreview = ({
+    source,
+    language,
+    path,
+    repoName,
+    revisionName,
+}: CodePreviewProps) => {
+    const editorRef = useRef<ReactCodeMirrorRef>(null);
+    const syntaxHighlighting = useSyntaxHighlightingExtension(language, editorRef.current?.view);
+    const [currentSelection, setCurrentSelection] = useState<SelectionRange>();
+    const keymapExtension = useKeymapExtension(editorRef.current?.view);
+    const [isEditorCreated, setIsEditorCreated] = useState(false);
+
+    const highlightRangeQuery = useNonEmptyQueryParam('highlightRange');
+    const highlightRange = useMemo(() => {
+        if (!highlightRangeQuery) {
+            return;
+        }
+
+        const rangeRegex = /^\d+:\d+,\d+:\d+$/;
+        if (!rangeRegex.test(highlightRangeQuery)) {
+            return;
+        }
+
+        const [start, end] = highlightRangeQuery.split(',').map((range) => {
+            return range.split(':').map((val) => parseInt(val, 10));
+        });
+
+        return {
+            start: {
+                line: start[0],
+                character: start[1],
+            },
+            end: {
+                line: end[0],
+                character: end[1],
+            }
+        }
+    }, [highlightRangeQuery]);
+
+    const extensions = useMemo(() => {
+        const highlightDecoration = Decoration.mark({
+            class: "cm-searchMatch-selected",
+          });
+
+        return [
+            syntaxHighlighting,
+            EditorView.lineWrapping,
+            keymapExtension,
+            search({
+                top: true,
+            }),
+            EditorView.updateListener.of((update: ViewUpdate) => {
+                if (update.selectionSet) {
+                    setCurrentSelection(update.state.selection.main);
+                }
+            }),
+            StateField.define<DecorationSet>({
+                create(state) {
+                    if (!highlightRange) {
+                        return Decoration.none;
+                    }
+
+                    const { start, end } = highlightRange;
+                    const from = state.doc.line(start.line).from + start.character - 1;
+                    const to = state.doc.line(end.line).from + end.character - 1;
+
+                    return Decoration.set([
+                        highlightDecoration.range(from, to),
+                    ]);
+                },
+                update(deco, tr) {
+                    return deco.map(tr.changes);
+                },
+                provide: (field) => EditorView.decorations.from(field),
+            }),
+        ];
+    }, [keymapExtension, syntaxHighlighting, highlightRange]);
+
+    useEffect(() => {
+        if (!highlightRange || !editorRef.current || !editorRef.current.state) {
+            return;
+        }
+
+        const doc = editorRef.current.state.doc;
+        const { start, end } = highlightRange;
+        const from = doc.line(start.line).from + start.character - 1;
+        const to = doc.line(end.line).from + end.character - 1;
+        const selection = EditorSelection.range(from, to);
+
+        editorRef.current.view?.dispatch({
+            effects: [
+                EditorView.scrollIntoView(selection, { y: "center" }),
+            ]
+        });
+        // @note: we need to include `isEditorCreated` in the dependency array since
+        // a race-condition can happen if the `highlightRange` is resolved before the
+        // editor is created.
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [highlightRange, isEditorCreated]);
+
+    const { theme } = useThemeNormalized();
+
+    return (
+        <ScrollArea className="h-full overflow-auto flex-1">
+            <CodeMirror
+                className="relative"
+                ref={editorRef}
+                onCreateEditor={() => {
+                    setIsEditorCreated(true);
+                }}
+                value={source}
+                extensions={extensions}
+                readOnly={true}
+                theme={theme === "dark" ? "dark" : "light"}
+            >
+                {editorRef.current && editorRef.current.view && currentSelection && (
+                    <EditorContextMenu
+                        view={editorRef.current.view}
+                        selection={currentSelection}
+                        repoName={repoName}
+                        path={path}
+                        revisionName={revisionName}
+                    />
+                )}
+            </CodeMirror>
+        </ScrollArea>
+    )
+}
+