Add git support

openSUSE · Dec 13, 2024 · 8f87ad4 · 8f87ad4
1 parent a1efd57
commit 8f87ad4
Show file tree

Hide file tree

Showing 17 changed files with 612 additions and 51 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -43,6 +43,10 @@ jobs:
           cpanm -n --installdeps .
           cpanm -n Test::Deep
           cpanm -n Devel::Cover::Report::Coveralls
+      - name: Git setup
+        run: |
+          git config --global user.name "GitHub Actions"
+          git config --global user.email "[email protected]"
       - name: Build assets
         env:
           NODE_ENV: production

diff --git a/README.md b/README.md
@@ -2,9 +2,8 @@
 
 ![Cavil](docs/images/cavil.png)
 
-  Cavil is a legal review and Software Bill of Materials (SBOM) system for the
-  [Open Build Service](https://openbuildservice.org). It is used in the development of openSUSE Tumbleweed,
-  openSUSE Leap, as well as SUSE Linux Enterprise.
+  Cavil is a legal review and Software Bill of Materials (SBOM) system. It is used in the development of
+  openSUSE Tumbleweed, openSUSE Leap, as well as SUSE Linux Enterprise.
 
 ## Features
 
@@ -16,7 +15,7 @@
 * Human reviews with approval/rejection workflow, and optional automatic approvals based on risk
 * Optional support for machine learning models to classify pattern matches
 * REST API for integration into existing source code management systems
-* Open Build Service integration via bots
+* [Open Build Service](https://openbuildservice.org) and [Gitea](https://gitea.com) integration via bots
 * OpenID Connect (OAuth 2.0) authentication
 
 **Important**: Note that most of the data used by Cavil has been curated by lawyers, but the generated reports do not
@@ -58,7 +57,8 @@ There are currently two example implementations for a companion server applicati
     $ sudo zypper in -C perl-Mojolicious perl-Mojolicious-Plugin-Webpack \
       perl-Mojo-Pg perl-Minion perl-File-Unpack perl-Cpanel-JSON-XS \
       perl-Spooky-Patterns-XS perl-Mojolicious-Plugin-OAuth2 perl-Mojo-JWT \
-      perl-BSD-Resource perl-Term-ProgressBar perl-Text-Glob
+      perl-BSD-Resource perl-Term-ProgressBar perl-Text-Glob perl-IPC-Run \
+      perl-Try-Tiny git git-lfs
     $ npm i
     $ npm run build
 

diff --git a/assets/vue/helpers/links.js b/assets/vue/helpers/links.js
@@ -21,6 +21,14 @@ export function externalLink(review) {
   if (link.substr(0, 4) === 'ibs#') {
     return `<a href='https://build.suse.de/request/show/${link.substr(4)}' target='_blank'>${link}</a>`;
   }
+  const sooMatch = link.match(/soo#([^!]+)!(\d+)/);
+  if (sooMatch !== null) {
+    return `<a href='https://src.opensuse.org/${sooMatch[1]}/pulls/${sooMatch[2]}' target='_blank'>${sooMatch[0]}</a>`;
+  }
+  const ssdMatch = link.match(/ssd#([^!]+)!(\d+)/);
+  if (ssdMatch !== null) {
+    return `<a href='https://src.suse.de/${ssdMatch[1]}/pulls/${ssdMatch[2]}' target='_blank'>${ssdMatch[0]}</a>`;
+  }
 
   return link;
 }

diff --git a/cpanfile b/cpanfile
@@ -4,6 +4,7 @@ requires 'Mojo::Pg',         '>= 4.27';
 requires 'Minion',           '>= 10.27';
 requires 'Cpanel::JSON::XS', '>= 4.09';
 requires 'File::Unpack2';
+requires 'IPC::Run';
 requires 'Spooky::Patterns::XS';
 requires 'Mojolicious::Plugin::OAuth2';
 requires 'Mojo::JWT';
@@ -13,6 +14,7 @@ requires 'Algorithm::Diff';
 requires 'IO::Socket::SSL', '>= 2.009';
 requires 'Text::Diff';
 requires 'Text::Glob';
+requires 'Try::Tiny';
 requires 'YAML::XS';
 requires 'JSON::Validator';
 requires 'Digest::SHA1';
diff --git a/docs/API.md b/docs/API.md
@@ -176,18 +176,20 @@ Create package.
 
 **Request parameters:**
 
-* `api` (required): Open Build Service API URL prefix.
+* `api` (required): Open Build Service API URL prefix or Git repository.
 
-* `project` (required): Open Build Service project name.
+* `project` (required): Open Build Service project name, if applicable.
 
-* `package` (required): Open Build Service package name.
+* `package` (required): Open Build Service or Git package name.
 
-* `rev` (optional): Revision to check out.
+* `rev` (optional): Open Build Service revision or Git commit to check out.
 
 * `created` (optional): Package creation timestamp.
 
-* `external_link` (optional): Short string describing the package source. Special values like `obs#123` and `ibs#123`
-                              result in links to `https://build.opensuse.org` and `https://build.suse.de`.
+* `external_link` (optional): Short string describing the package source. Special values like `obs#123`, `ibs#123`,
+                              `soo#org/package!123` and `ssd#org/package!123` result in links to
+                              `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and
+                              `https://src.suse.de`.
 
 * `priority` (optional): Priority of this package review.
 
@@ -253,8 +255,10 @@ Re-import package. Usually used to reopen a review after it has already been obs
 
 * `priority` (optional): Priority of this package review.
 
-* `external_link` (optional): Short string describing the package source. Special values like `obs#123` and `ibs#123`
-                              result in links to `https://build.opensuse.org` and `https://build.suse.de`.
+* `external_link` (optional): Short string describing the package source. Special values like `obs#123`, `ibs#123`,
+                              `soo#org/package!123` and `ssd#org/package!123` result in links to
+                              `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and
+                              `https://src.suse.de`.
 
 ```
 POST /packages/import/23
@@ -316,8 +320,10 @@ Create request for package.
 
 * `package` (required): Package id.
 
-* `external_link` (required): Short string describing the package source. Special values like `obs#123` and `ibs#123`
-                              result in links to `https://build.opensuse.org` and `https://build.suse.de`.
+* `external_link` (required): Short string describing the package source. Special values like `obs#123`, `ibs#123`,
+                              `soo#org/package!123` and `ssd#org/package!123` result in links to
+                              `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and
+                              `https://src.suse.de`.
 
 ```
 POST /requests
@@ -385,8 +391,10 @@ Delete review requests.
 
 **Request parameters:**
 
-* `external_link` (required): Short string describing the package source. Special values like `obs#123` and `ibs#123`
-                              result in links to `https://build.opensuse.org` and `https://build.suse.de`.
+* `external_link` (required): Short string describing the package source. Special values like `obs#123`, `ibs#123`,
+                              `soo#org/package!123` and `ssd#org/package!123` result in links to
+                              `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and
+                              `https://src.suse.de`.
 
 ```
 DELETE /requests

diff --git a/lib/Cavil.pm b/lib/Cavil.pm
@@ -18,6 +18,7 @@ use Mojo::Base 'Mojolicious', -signatures;
 
 use Mojo::Pg;
 use Cavil::Classifier;
+use Cavil::Git;
 use Cavil::Model::IgnoredFiles;
 use Cavil::Model::Packages;
 use Cavil::Model::Patterns;
@@ -33,6 +34,7 @@ use Scalar::Util 'weaken';
 use Mojo::File qw(path);
 
 has classifier => sub { Cavil::Classifier->new };
+has git        => sub { Cavil::Git->new };
 has obs        => sub { Cavil::OBS->new };
 has spdx       => sub ($self) {
   my $spdx = Cavil::SPDX->new(app => $self);

diff --git a/lib/Cavil/Command/git.pm b/lib/Cavil/Command/git.pm
@@ -0,0 +1,86 @@
+# Copyright (C) 2024 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package Cavil::Command::git;
+use Mojo::Base 'Mojolicious::Command', -signatures;
+
+use Mojo::File qw(path);
+use Mojo::Util qw(getopt);
+
+
+has description => 'Import git sources';
+has usage       => sub ($self) { $self->extract_usage };
+
+sub run ($self, @args) {
+  getopt \@args, 'e|external-link=s' => \my $link, 'i|import' => \my $import;
+
+  my $url  = shift @args;
+  my $pkg  = shift @args;
+  my $hash = shift @args;
+
+  die "URL is required.\n"     unless $url;
+  die "PACKAGE is required.\n" unless $pkg;
+  die "HASH is required.\n"    unless $hash;
+
+  return print STDOUT "Nothing to do\n" unless $import;
+
+  # Index
+  my $app  = $self->app;
+  my $user = $app->users->licensedigger;
+  my $pkgs = $app->packages;
+  my $obj  = $pkgs->find_by_name_and_md5($pkg, $hash);
+  if (!$obj) {
+    my $id = $pkgs->add(
+      name            => $pkg,
+      checkout_dir    => $hash,
+      api_url         => $url,
+      requesting_user => $user->{id},
+      project         => '',
+      priority        => 1,
+      package         => $pkg,
+      srcmd5          => $hash,
+      type            => 'git'
+    );
+    $obj = $pkgs->find($id);
+  }
+  $obj->{external_link} = $link // $obj->{external_link} // 'git-command';
+  $obj->{obsolete}      = 0;
+  $pkgs->update($obj);
+  my $job = $pkgs->git_import($obj->{id}, {url => $url, pkg => $pkg, hash => $hash, priority => 9}, 9);
+
+  print STDOUT "Triggered git_import job $job\n";
+}
+
+1;
+
+=encoding utf8
+
+=head1 NAME
+
+Cavil::Command::git - Cavil git command
+
+=head1 SYNOPSIS
+
+  Usage: APPLICATION git [URL] [PACKAGE] [HASH]
+
+    script/cavil git https://src.opensuse.org/pool/perl-Mojolicious.git perl-Mojolicious \
+                     242511548e0cdcf17b6321738e2d8b6a3b79d41775c4a867f03b384a284d9168 -i
+
+  Options:
+    -e, --external-link <link>   External link to the request
+    -i, --import                 Import and index package from git
+    -h, --help                   Show this summary of available options
+
+=cut
diff --git a/lib/Cavil/Controller/Queue.pm b/lib/Cavil/Controller/Queue.pm
@@ -20,17 +20,29 @@ use Mojo::File 'path';
 
 sub create_package ($self) {
   my $validation = $self->validation;
+
+  $validation->optional('type')->in('obs', 'git');
+  my $type = $validation->param('type') || 'obs';
+
+  if ($type eq 'git') {
+    $validation->required('rev')->like(qr/^[a-f0-9]+$/i);
+    $validation->optional('project');
+  }
+  else {
+    $validation->optional('rev')->like(qr/^[a-f0-9]+$/i);
+    $validation->required('project');
+  }
+
   $validation->required('api')->like(qr!^https?://.+!i);
-  $validation->required('project');
   $validation->required('package');
-  $validation->optional('rev')->like(qr/^[a-f0-9]+$/i);
   $validation->optional('created');
   $validation->optional('external_link');
   $validation->optional('priority')->like(qr/^\d+$/);
+
   return $self->reply->json_validation_error if $validation->has_error;
 
   my $api     = $validation->param('api');
-  my $project = $validation->param('project');
+  my $project = $validation->param('project') // '';
   my $pkg     = $validation->param('package');
   my $rev     = $validation->param('rev');
   my $created = $validation->param('created');
@@ -39,18 +51,25 @@ sub create_package ($self) {
 
   my $app    = $self->app;
   my $config = $app->config;
-  my $obs    = $app->obs;
+
+  my ($srcpkg, $srcmd5, $verifymd5);
+  if ($type eq 'git') {
+    ($srcpkg, $srcmd5, $verifymd5) = ($pkg, $rev, $rev);
+  }
 
   # Get package infomation, rev may be pointing to link, so we need the
   # canonical srcmd5
-  my $info = eval { $obs->package_info($api, $project, $pkg, {rev => $rev}) };
-  unless ($info && $info->{verifymd5}) {
-    $self->_log("Couldn't get package info", $api, $project, $pkg, $rev, $@);
-    return $self->render(json => {error => 'Package not found'}, status => 404);
+  else {
+    my $obs  = $app->obs;
+    my $info = eval { $obs->package_info($api, $project, $pkg, {rev => $rev}) };
+    unless ($info && $info->{verifymd5}) {
+      $self->_log("Couldn't get package info", $api, $project, $pkg, $rev, $@);
+      return $self->render(json => {error => 'Package not found'}, status => 404);
+    }
+    ($srcpkg, $srcmd5, $verifymd5) = @{$info}{qw(package srcmd5 verifymd5)};
   }
-  my ($srcpkg, $srcmd5, $verifymd5) = @{$info}{qw(package srcmd5 verifymd5)};
 
-  # Check if we need to import from OBS
+  # Check if we need to import
   my $dir    = path($config->{checkout_dir}, $srcpkg, $verifymd5);
   my $create = !-e $dir;
 
@@ -68,6 +87,7 @@ sub create_package ($self) {
       package         => $pkg,
       created         => $created,
       srcmd5          => $srcmd5,
+      type            => $type
     );
     $obj = $pkgs->find($id);
   }
@@ -79,21 +99,28 @@ sub create_package ($self) {
   $obj->{obsolete} = 0;
   $pkgs->update($obj);
   if ($create) {
-    $pkgs->obs_import(
-      $obj->{id},
-      {
-        api           => $api,
-        project       => $project,
-        pkg           => $pkg,
-        srcpkg        => $srcpkg,
-        rev           => $rev,
-        srcmd5        => $srcmd5,
-        verifymd5     => $verifymd5,
-        external_link => $obj->{external_link},
-        priority      => $prio
-      },
-      $prio + 10
-    );
+    if ($type eq 'git') {
+      $pkgs->git_import($obj->{id},
+        {url => $api, pkg => $pkg, hash => $rev, external_link => $obj->{external_link}, priority => $prio},
+        $prio + 10);
+    }
+    else {
+      $pkgs->obs_import(
+        $obj->{id},
+        {
+          api           => $api,
+          project       => $project,
+          pkg           => $pkg,
+          srcpkg        => $srcpkg,
+          rev           => $rev,
+          srcmd5        => $srcmd5,
+          verifymd5     => $verifymd5,
+          external_link => $obj->{external_link},
+          priority      => $prio
+        },
+        $prio + 10
+      );
+    }
   }
 
   $self->render(json => {saved => $obj});