From 9b5ec71266833c6986d81422f3c7f7b3a181651f Mon Sep 17 00:00:00 2001 From: Sebastian Riedel Date: Tue, 3 Dec 2024 16:02:28 +0100 Subject: [PATCH] Add git support --- .github/workflows/linux.yml | 4 + README.md | 10 +- assets/vue/helpers/links.js | 8 ++ cpanfile | 2 + docs/API.md | 32 ++++--- lib/Cavil.pm | 2 + lib/Cavil/Command/git.pm | 86 +++++++++++++++++ lib/Cavil/Controller/Queue.pm | 77 +++++++++++----- lib/Cavil/Git.pm | 58 ++++++++++++ lib/Cavil/Model/Packages.pm | 21 ++++- lib/Cavil/Task/Import.pm | 29 ++++++ lib/Cavil/Util.pm | 27 +++++- migrations/cavil.sql | 3 + t/git.t | 169 ++++++++++++++++++++++++++++++++++ t/util.t | 15 ++- 15 files changed, 492 insertions(+), 51 deletions(-) create mode 100644 lib/Cavil/Command/git.pm create mode 100644 lib/Cavil/Git.pm create mode 100644 t/git.t diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2cc766b385..cd9ac3cc1e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -43,6 +43,10 @@ jobs: cpanm -n --installdeps . cpanm -n Test::Deep cpanm -n Devel::Cover::Report::Coveralls + - name: Git setup + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "nospam@github.com" - name: Build assets env: NODE_ENV: production diff --git a/README.md b/README.md index f6d11d9edd..6723615e0a 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ ![Cavil](docs/images/cavil.png) - Cavil is a legal review and Software Bill of Materials (SBOM) system for the - [Open Build Service](https://openbuildservice.org). It is used in the development of openSUSE Tumbleweed, - openSUSE Leap, as well as SUSE Linux Enterprise. + Cavil is a legal review and Software Bill of Materials (SBOM) system. It is used in the development of + openSUSE Tumbleweed, openSUSE Leap, as well as SUSE Linux Enterprise. ## Features @@ -16,7 +15,7 @@ * Human reviews with approval/rejection workflow, and optional automatic approvals based on risk * Optional support for machine learning models to classify pattern matches * REST API for integration into existing source code management systems -* Open Build Service integration via bots +* [Open Build Service](https://openbuildservice.org) and [Gitea](https://gitea.com) integration via bots * OpenID Connect (OAuth 2.0) authentication **Important**: Note that most of the data used by Cavil has been curated by lawyers, but the generated reports do not @@ -58,7 +57,8 @@ There are currently two example implementations for a companion server applicati $ sudo zypper in -C perl-Mojolicious perl-Mojolicious-Plugin-Webpack \ perl-Mojo-Pg perl-Minion perl-File-Unpack perl-Cpanel-JSON-XS \ perl-Spooky-Patterns-XS perl-Mojolicious-Plugin-OAuth2 perl-Mojo-JWT \ - perl-BSD-Resource perl-Term-ProgressBar perl-Text-Glob + perl-BSD-Resource perl-Term-ProgressBar perl-Text-Glob perl-IPC-Run \ + perl-Try-Tiny git git-lfs $ npm i $ npm run build diff --git a/assets/vue/helpers/links.js b/assets/vue/helpers/links.js index b3a4362261..b44b4e89fb 100644 --- a/assets/vue/helpers/links.js +++ b/assets/vue/helpers/links.js @@ -21,6 +21,14 @@ export function externalLink(review) { if (link.substr(0, 4) === 'ibs#') { return `${link}`; } + const sooMatch = link.match(/soo#([^!]+)!(\d+)/); + if (sooMatch !== null) { + return `${sooMatch[0]}`; + } + const ssdMatch = link.match(/ssd#([^!]+)!(\d+)/); + if (ssdMatch !== null) { + return `${ssdMatch[0]}`; + } return link; } diff --git a/cpanfile b/cpanfile index 4e213e19ff..50cb030663 100644 --- a/cpanfile +++ b/cpanfile @@ -4,6 +4,7 @@ requires 'Mojo::Pg', '>= 4.27'; requires 'Minion', '>= 10.27'; requires 'Cpanel::JSON::XS', '>= 4.09'; requires 'File::Unpack2'; +requires 'IPC::Run'; requires 'Spooky::Patterns::XS'; requires 'Mojolicious::Plugin::OAuth2'; requires 'Mojo::JWT'; @@ -13,6 +14,7 @@ requires 'Algorithm::Diff'; requires 'IO::Socket::SSL', '>= 2.009'; requires 'Text::Diff'; requires 'Text::Glob'; +requires 'Try::Tiny'; requires 'YAML::XS'; requires 'JSON::Validator'; requires 'Digest::SHA1'; diff --git a/docs/API.md b/docs/API.md index 2d2fe6a0c3..03468df9d7 100644 --- a/docs/API.md +++ b/docs/API.md @@ -176,18 +176,20 @@ Create package. **Request parameters:** -* `api` (required): Open Build Service API URL prefix. +* `api` (required): Open Build Service API URL prefix or Git repository. -* `project` (required): Open Build Service project name. +* `project` (required): Open Build Service project name, if applicable. -* `package` (required): Open Build Service package name. +* `package` (required): Open Build Service or Git package name. -* `rev` (optional): Revision to check out. +* `rev` (optional): Open Build Service revision or Git commit to check out. * `created` (optional): Package creation timestamp. -* `external_link` (optional): Short string describing the package source. Special values like `obs#123` and `ibs#123` - result in links to `https://build.opensuse.org` and `https://build.suse.de`. +* `external_link` (optional): Short string describing the package source. Special values like `obs#123`, `ibs#123`, + `soo#org/package!123` and `ssd#org/package!123` result in links to + `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and + `https://src.suse.de`. * `priority` (optional): Priority of this package review. @@ -253,8 +255,10 @@ Re-import package. Usually used to reopen a review after it has already been obs * `priority` (optional): Priority of this package review. -* `external_link` (optional): Short string describing the package source. Special values like `obs#123` and `ibs#123` - result in links to `https://build.opensuse.org` and `https://build.suse.de`. +* `external_link` (optional): Short string describing the package source. Special values like `obs#123`, `ibs#123`, + `soo#org/package!123` and `ssd#org/package!123` result in links to + `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and + `https://src.suse.de`. ``` POST /packages/import/23 @@ -316,8 +320,10 @@ Create request for package. * `package` (required): Package id. -* `external_link` (required): Short string describing the package source. Special values like `obs#123` and `ibs#123` - result in links to `https://build.opensuse.org` and `https://build.suse.de`. +* `external_link` (required): Short string describing the package source. Special values like `obs#123`, `ibs#123`, + `soo#org/package!123` and `ssd#org/package!123` result in links to + `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and + `https://src.suse.de`. ``` POST /requests @@ -385,8 +391,10 @@ Delete review requests. **Request parameters:** -* `external_link` (required): Short string describing the package source. Special values like `obs#123` and `ibs#123` - result in links to `https://build.opensuse.org` and `https://build.suse.de`. +* `external_link` (required): Short string describing the package source. Special values like `obs#123`, `ibs#123`, + `soo#org/package!123` and `ssd#org/package!123` result in links to + `https://build.opensuse.org`, `https://build.suse.de`, `https://src.opensuse.org` and + `https://src.suse.de`. ``` DELETE /requests diff --git a/lib/Cavil.pm b/lib/Cavil.pm index 978c6123cc..4b58aeee2e 100644 --- a/lib/Cavil.pm +++ b/lib/Cavil.pm @@ -18,6 +18,7 @@ use Mojo::Base 'Mojolicious', -signatures; use Mojo::Pg; use Cavil::Classifier; +use Cavil::Git; use Cavil::Model::IgnoredFiles; use Cavil::Model::Packages; use Cavil::Model::Patterns; @@ -33,6 +34,7 @@ use Scalar::Util 'weaken'; use Mojo::File qw(path); has classifier => sub { Cavil::Classifier->new }; +has git => sub { Cavil::Git->new }; has obs => sub { Cavil::OBS->new }; has spdx => sub ($self) { my $spdx = Cavil::SPDX->new(app => $self); diff --git a/lib/Cavil/Command/git.pm b/lib/Cavil/Command/git.pm new file mode 100644 index 0000000000..f25d2eacb7 --- /dev/null +++ b/lib/Cavil/Command/git.pm @@ -0,0 +1,86 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package Cavil::Command::git; +use Mojo::Base 'Mojolicious::Command', -signatures; + +use Mojo::File qw(path); +use Mojo::Util qw(getopt); + + +has description => 'Import git sources'; +has usage => sub ($self) { $self->extract_usage }; + +sub run ($self, @args) { + getopt \@args, 'e|external-link=s' => \my $link, 'i|import' => \my $import; + + my $url = shift @args; + my $pkg = shift @args; + my $hash = shift @args; + + die "URL is required.\n" unless $url; + die "PACKAGE is required.\n" unless $pkg; + die "HASH is required.\n" unless $hash; + + return print STDOUT "Nothing to do\n" unless $import; + + # Index + my $app = $self->app; + my $user = $app->users->licensedigger; + my $pkgs = $app->packages; + my $obj = $pkgs->find_by_name_and_md5($pkg, $hash); + if (!$obj) { + my $id = $pkgs->add( + name => $pkg, + checkout_dir => $hash, + api_url => $url, + requesting_user => $user->{id}, + project => '', + priority => 1, + package => $pkg, + srcmd5 => $hash, + type => 'git' + ); + $obj = $pkgs->find($id); + } + $obj->{external_link} = $link // $obj->{external_link} // 'git-command'; + $obj->{obsolete} = 0; + $pkgs->update($obj); + my $job = $pkgs->git_import($obj->{id}, {url => $url, pkg => $pkg, hash => $hash, priority => 9}, 9); + + print STDOUT "Triggered git_import job $job\n"; +} + +1; + +=encoding utf8 + +=head1 NAME + +Cavil::Command::git - Cavil git command + +=head1 SYNOPSIS + + Usage: APPLICATION git [URL] [PACKAGE] [HASH] + + script/cavil git https://src.opensuse.org/pool/perl-Mojolicious.git perl-Mojolicious \ + 242511548e0cdcf17b6321738e2d8b6a3b79d41775c4a867f03b384a284d9168 -i + + Options: + -e, --external-link External link to the request + -i, --import Import and index package from git + -h, --help Show this summary of available options + +=cut diff --git a/lib/Cavil/Controller/Queue.pm b/lib/Cavil/Controller/Queue.pm index 0619c0304c..db12183567 100644 --- a/lib/Cavil/Controller/Queue.pm +++ b/lib/Cavil/Controller/Queue.pm @@ -20,17 +20,29 @@ use Mojo::File 'path'; sub create_package ($self) { my $validation = $self->validation; + + $validation->optional('type')->in('obs', 'git'); + my $type = $validation->param('type') || 'obs'; + + if ($type eq 'git') { + $validation->required('rev')->like(qr/^[a-f0-9]+$/i); + $validation->optional('project'); + } + else { + $validation->optional('rev')->like(qr/^[a-f0-9]+$/i); + $validation->required('project'); + } + $validation->required('api')->like(qr!^https?://.+!i); - $validation->required('project'); $validation->required('package'); - $validation->optional('rev')->like(qr/^[a-f0-9]+$/i); $validation->optional('created'); $validation->optional('external_link'); $validation->optional('priority')->like(qr/^\d+$/); + return $self->reply->json_validation_error if $validation->has_error; my $api = $validation->param('api'); - my $project = $validation->param('project'); + my $project = $validation->param('project') // ''; my $pkg = $validation->param('package'); my $rev = $validation->param('rev'); my $created = $validation->param('created'); @@ -39,18 +51,25 @@ sub create_package ($self) { my $app = $self->app; my $config = $app->config; - my $obs = $app->obs; + + my ($srcpkg, $srcmd5, $verifymd5); + if ($type eq 'git') { + ($srcpkg, $srcmd5, $verifymd5) = ($pkg, $rev, $rev); + } # Get package infomation, rev may be pointing to link, so we need the # canonical srcmd5 - my $info = eval { $obs->package_info($api, $project, $pkg, {rev => $rev}) }; - unless ($info && $info->{verifymd5}) { - $self->_log("Couldn't get package info", $api, $project, $pkg, $rev, $@); - return $self->render(json => {error => 'Package not found'}, status => 404); + else { + my $obs = $app->obs; + my $info = eval { $obs->package_info($api, $project, $pkg, {rev => $rev}) }; + unless ($info && $info->{verifymd5}) { + $self->_log("Couldn't get package info", $api, $project, $pkg, $rev, $@); + return $self->render(json => {error => 'Package not found'}, status => 404); + } + ($srcpkg, $srcmd5, $verifymd5) = @{$info}{qw(package srcmd5 verifymd5)}; } - my ($srcpkg, $srcmd5, $verifymd5) = @{$info}{qw(package srcmd5 verifymd5)}; - # Check if we need to import from OBS + # Check if we need to import my $dir = path($config->{checkout_dir}, $srcpkg, $verifymd5); my $create = !-e $dir; @@ -68,6 +87,7 @@ sub create_package ($self) { package => $pkg, created => $created, srcmd5 => $srcmd5, + type => $type ); $obj = $pkgs->find($id); } @@ -79,21 +99,28 @@ sub create_package ($self) { $obj->{obsolete} = 0; $pkgs->update($obj); if ($create) { - $pkgs->obs_import( - $obj->{id}, - { - api => $api, - project => $project, - pkg => $pkg, - srcpkg => $srcpkg, - rev => $rev, - srcmd5 => $srcmd5, - verifymd5 => $verifymd5, - external_link => $obj->{external_link}, - priority => $prio - }, - $prio + 10 - ); + if ($type eq 'git') { + $pkgs->git_import($obj->{id}, + {url => $api, pkg => $pkg, hash => $rev, external_link => $obj->{external_link}, priority => $prio}, + $prio + 10); + } + else { + $pkgs->obs_import( + $obj->{id}, + { + api => $api, + project => $project, + pkg => $pkg, + srcpkg => $srcpkg, + rev => $rev, + srcmd5 => $srcmd5, + verifymd5 => $verifymd5, + external_link => $obj->{external_link}, + priority => $prio + }, + $prio + 10 + ); + } } $self->render(json => {saved => $obj}); diff --git a/lib/Cavil/Git.pm b/lib/Cavil/Git.pm new file mode 100644 index 0000000000..5c1dfd2fe3 --- /dev/null +++ b/lib/Cavil/Git.pm @@ -0,0 +1,58 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package Cavil::Git; +use Mojo::Base -base, -signatures; + +use Carp 'croak'; +use Cavil::Util qw(run_cmd); +use Mojo::File qw(path); +use Mojo::Util qw(dumper); + +use constant DEBUG => $ENV{CAVIL_GIT_DEBUG} || 0; + +has config => sub { {} }; + +sub git_cmd ($self, $dir, $args) { + my $config = $self->config; + my $git = $config->{git} || 'git'; + + # Prevent password prompts + local $ENV{GIT_SSH_COMMAND} = 'ssh -oBatchMode=yes'; + + my @cmd = ($git, @$args); + my $result = run_cmd($dir, \@cmd); + + warn dumper({cmd => \@cmd, result => $result}) if DEBUG; + croak qq/Git command "@{[join(' ', @cmd)]}" failed: $result->{stderr}/ if !$result->{status} || $result->{exit_code}; +} + +sub download_source ($self, $url, $dir, $options = {}) { + my $hash = $options->{hash} || 'main'; + + # Clean up directory in case of failed previous checkouts + $dir = path($dir)->remove_tree->make_path; + + $self->git_cmd($dir, ['init', length($hash) == 64 ? '--object-format=sha256' : ()]); + $self->git_cmd($dir, ['remote', 'add', 'r', $url]); + $self->git_cmd($dir, ['fetch', '--depth', '1', 'r', $hash]); + $self->git_cmd($dir, ['checkout', $hash]); + + chmod 0755, $dir; + chmod 0644, $_ for $dir->list_tree->each; + $dir->child('.git')->remove_tree; +} + +1; diff --git a/lib/Cavil/Model/Packages.pm b/lib/Cavil/Model/Packages.pm index baa1de05f4..ef325d1de4 100644 --- a/lib/Cavil/Model/Packages.pm +++ b/lib/Cavil/Model/Packages.pm @@ -24,9 +24,14 @@ has [qw(checkout_dir log minion pg)]; sub add ($self, %args) { - my $db = $self->pg->db; - my $source - = {api_url => $args{api_url}, project => $args{project}, package => $args{package}, srcmd5 => $args{srcmd5}}; + my $db = $self->pg->db; + my $source = { + api_url => $args{api_url}, + project => $args{project}, + package => $args{package}, + srcmd5 => $args{srcmd5}, + type => $args{type} // 'obs' + }; my $source_id = $db->insert('bot_sources', $source, {returning => 'id'})->hash->{id}; my $pkg = { @@ -394,6 +399,16 @@ sub name_suggestions ($self, $partial) { )->arrays->flatten->to_array; } +sub git_import ($self, $id, $data, $priority = 5) { + my $pkg = $self->find($id); + return $self->minion->enqueue( + git_import => [$id, $data] => { + priority => $priority, + notes => {external_link => $pkg->{external_link}, package => $pkg->{name}, "pkg_$id" => 1} + } + ); +} + sub obs_import ($self, $id, $data, $priority = 5) { my $pkg = $self->find($id); return $self->minion->enqueue( diff --git a/lib/Cavil/Task/Import.pm b/lib/Cavil/Task/Import.pm index 77daa9ecb8..2e3cbd3ec7 100644 --- a/lib/Cavil/Task/Import.pm +++ b/lib/Cavil/Task/Import.pm @@ -21,6 +21,7 @@ use Mojo::File qw(path); use Cavil::Util qw(request_id_from_external_link); sub register ($self, $app, $config) { + $app->minion->add_task(git_import => \&_git); $app->minion->add_task(obs_import => \&_obs); } @@ -33,6 +34,34 @@ sub _embargo ($job, $id, $data) { $app->packages->update({id => $id, embargoed => $embargoed}); } +sub _git ($job, $id, $data) { + my $app = $job->app; + my $minion = $app->minion; + my $log = $app->log; + my $pkgs = $app->packages; + + # Protect from race conditions + return $job->finish("Package $id is already being processed") + unless my $guard = $minion->guard("processing_pkg_$id", 172800); + + my $checkout_dir = $app->config->{checkout_dir}; + my ($pkg, $url, $hash, $priority) = @{$data}{qw(pkg url hash priority)}; + my $dir = path($checkout_dir, $pkg, $hash); + + my $git = $app->git; + eval { $git->download_source($url, $dir, {hash => $hash}) }; + if ($@) { + $dir->remove_tree; + die $@; + } + $pkgs->imported($id); + $log->info("[$id] Imported $dir"); + + # Next step + undef $guard; + $pkgs->unpack($id, 8, [$job->id]); +} + sub _obs ($job, $id, $data) { my $app = $job->app; my $minion = $app->minion; diff --git a/lib/Cavil/Util.pm b/lib/Cavil/Util.pm index 2fb6b98aaf..c4a04b83f9 100644 --- a/lib/Cavil/Util.pm +++ b/lib/Cavil/Util.pm @@ -18,19 +18,21 @@ use Mojo::Base -strict, -signatures; use Carp 'croak'; use Exporter 'import'; -use Encode qw(from_to decode); +use Encode qw(from_to decode); +use IPC::Run (); use Mojo::Util; use Mojo::File qw(path tempfile); use POSIX 'ceil'; use Spooky::Patterns::XS; use Text::Glob 'glob_to_regex'; +use Try::Tiny; $Text::Glob::strict_wildcard_slash = 0; our @EXPORT_OK = ( qw(buckets file_and_checksum slurp_and_decode load_ignored_files lines_context obs_ssh_auth paginate), - qw(parse_exclude_file pattern_checksum pattern_matches read_lines request_id_from_external_link snippet_checksum), - qw(ssh_sign) + qw(parse_exclude_file pattern_checksum pattern_matches read_lines request_id_from_external_link run_cmd), + qw(snippet_checksum ssh_sign) ); my $MAX_FILE_SIZE = 30000; @@ -87,6 +89,25 @@ sub pattern_checksum ($text) { return $ctx->hex; } +sub run_cmd ($dir, $cmd) { + my $cwd = path; + chdir $dir; + my $guard = Mojo::Util::scope_guard sub { chdir $cwd }; + + try { + my ($stdin, $stdout, $stderr) = ('', '', ''); + my $success = IPC::Run::run($cmd, \$stdin, \$stdout, \$stderr); + my $status = $?; + return {status => $success, exit_code => $status >> 8, stdout => $stdout, stderr => $stderr}; + } + catch { + return {status => 0, exit_code => undef, stdout => '', stderr => $_ // 'Unknown error'}; + } + finally { + undef $guard; + }; +} + sub snippet_checksum ($text) { my $ctx = Spooky::Patterns::XS::init_hash(0, 0); $ctx->add($text); diff --git a/migrations/cavil.sql b/migrations/cavil.sql index c45a2817a5..135519a0da 100644 --- a/migrations/cavil.sql +++ b/migrations/cavil.sql @@ -235,3 +235,6 @@ ALTER TABLE bot_packages ADD COLUMN notice text; ALTER TABLE bot_packages ADD COLUMN embargoed boolean DEFAULT false NOT NULL; CREATE INDEX ON bot_packages(embargoed); ALTER TABLE snippets ADD COLUMN package int REFERENCES bot_packages(id) ON DELETE SET NULL; + +-- 25 up +ALTER TABLE bot_sources ADD COLUMN type text DEFAULT 'obs' NOT NULL; diff --git a/t/git.t b/t/git.t new file mode 100644 index 0000000000..774d38b424 --- /dev/null +++ b/t/git.t @@ -0,0 +1,169 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +use Mojo::Base -strict, -signatures; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use Test::More; +use Test::Mojo; +use Mojo::File qw(tempdir); +use Cavil::Util qw(run_cmd); +use Cavil::Git; +use Cavil::Test; + +plan skip_all => 'set TEST_ONLINE to enable this test' unless $ENV{TEST_ONLINE}; + +my $git = Cavil::Git->new; + +subtest 'Local git' => sub { + my $src_dir = tempdir; + $git->git_cmd($src_dir, ['init']); + my $file = $src_dir->child('test.txt')->spew('one'); + $git->git_cmd($src_dir, ['add', '.']); + $git->git_cmd($src_dir, ['commit', '-m', 'commit one']); + my $first_hash = run_cmd($src_dir, ['git', 'rev-parse', 'HEAD'])->{stdout}; + chomp $first_hash; + $file->spew('two'); + $git->git_cmd($src_dir, ['commit', '-am', 'commit two']); + my $second_hash = run_cmd($src_dir, ['git', 'rev-parse', 'HEAD'])->{stdout}; + chomp $second_hash; + $file->spew('three'); + $git->git_cmd($src_dir, ['commit', '-am', 'commit three']); + my $third_hash = run_cmd($src_dir, ['git', 'rev-parse', 'HEAD'])->{stdout}; + chomp $third_hash; + + subtest 'Download second commit' => sub { + my $dir = tempdir; + $git->download_source($src_dir, $dir, {hash => $second_hash}); + my $files = $dir->list_tree({hidden => 1}); + is scalar @$files, 1, '1 file in the tree'; + is $dir->child('test.txt')->slurp, 'two', 'right content'; + }; + + subtest 'Download first commit' => sub { + my $dir = tempdir; + $git->download_source($src_dir, $dir, {hash => $first_hash}); + my $files = $dir->list_tree({hidden => 1}); + is scalar @$files, 1, '1 file in the tree'; + is $dir->child('test.txt')->slurp, 'one', 'right content'; + }; + + subtest 'Download third commit' => sub { + my $dir = tempdir; + $git->download_source($src_dir, $dir, {hash => $third_hash}); + my $files = $dir->list_tree({hidden => 1}); + is scalar @$files, 1, '1 file in the tree'; + is $dir->child('test.txt')->slurp, 'three', 'right content'; + }; +}; + +subtest 'Live git server tests' => sub { + plan skip_all => 'set TEST_LIVE to run live tests' unless $ENV{TEST_LIVE}; + + subtest 'Source download from GitHub (SSH)' => sub { + my $dir = tempdir; + $git->download_source('git@github.com:openSUSE/cavil.git', + $dir, {hash => 'a1efd571deeb430d03d59075ae9c76a7e79c3988'}); + my $files = $dir->list_tree({hidden => 1}); + is scalar @$files, 300, '300 files in the tree'; + is $files->[0]->basename, '.eslintrc.json', 'first file is .eslintrc.json'; + is $files->[-1]->basename, 'webpack.config.js', 'last file is webpack.config.js'; + like $files->[-1]->slurp, qr/export default config/s, 'right content'; + }; + + subtest 'Source download from Gitea (HTTPS)' => sub { + my $dir = tempdir; + $git->download_source('https://src.opensuse.org/pool/perl-Mojolicious.git', + $dir, {hash => '242511548e0cdcf17b6321738e2d8b6a3b79d41775c4a867f03b384a284d9168'}); + my $files = $dir->list_tree({hidden => 1}); + is scalar @$files, 6, '6 files in the tree'; + is $files->[0]->basename, '.gitattributes', 'first file is .gitattributes'; + is $files->[-1]->basename, 'perl-Mojolicious.spec', 'last file is perl-Mojolicious.spec'; + like $files->[-1]->slurp, qr/Version:\s+9\.380\.0/s, 'right content'; + }; + + subtest 'Source download from GitHub (bad remote ref)' => sub { + my $dir = tempdir; + eval { + $git->download_source('git@github.com:openSUSE/cavil.git', + $dir, {hash => 'b1efd571deeb430d03d59075ae9c76a7e79c39899'}); + }; + like $@, qr/Git command "git fetch .+99" failed:.+remote ref b1efd571deeb430d03d59075ae9c76a7e79c39899/s, + 'right error'; + }; + + subtest 'Bot API (with Minion background jobs)' => sub { + my $cavil_test = Cavil::Test->new(online => $ENV{TEST_ONLINE}, schema => 'git_import_test'); + my $config = $cavil_test->default_config; + my $t = Test::Mojo->new(Cavil => $config); + $cavil_test->no_fixtures($t->app); + + my $headers = {Authorization => "Token $config->{tokens}[0]"}; + + subtest 'Validation errors' => sub { + $t->post_ok('/packages')->status_is(403); + $t->post_ok('/packages', $headers)->status_is(400) + ->json_is({error => 'Invalid request parameters (api, package, project)'}); + $t->post_ok('/packages', $headers, form => {type => 'git'})->status_is(400) + ->json_is({error => 'Invalid request parameters (api, package, rev)'}); + }; + + subtest 'Standard import' => sub { + $t->post_ok( + '/packages' => $headers => form => { + api => 'https://src.opensuse.org/pool/perl-Mojolicious.git', + package => 'perl-Mojolicious', + rev => '242511548e0cdcf17b6321738e2d8b6a3b79d41775c4a867f03b384a284d9168', + type => 'git' + } + )->status_is(200)->json_is('/saved/id' => 1); + ok !$t->app->packages->is_imported(1), 'not imported yet'; + $t->get_ok('/package/1', $headers)->status_is(200)->json_is('/state' => 'new')->json_is('/imported' => undef); + + my $minion = $t->app->minion; + my $worker = $minion->worker->register; + my $job_id = $minion->jobs({tasks => ['git_import']})->next->{id}; + ok my $job = $worker->dequeue(0, {id => $job_id}), 'job dequeued'; + is $job->execute, undef, 'no error'; + ok $minion->lock('processing_pkg_1', 0), 'lock no longer exists'; + $worker->unregister; + ok $t->app->packages->is_imported(1), 'imported'; + + $t->get_ok('/package/1', $headers)->status_is(200)->json_is('/state' => 'new')->json_like('/imported' => qr/\d/); + unlike $minion->job($job_id)->info->{result}, qr/Package \d+ is already being processed/, 'no race condition'; + }; + + subtest 'Prevent import race condition' => sub { + my $minion = $t->app->minion; + my $worker = $minion->worker->register; + my $job_id = $minion->jobs({tasks => ['git_import']})->next->{id}; + ok $minion->job($job_id)->retry, 'import job retried'; + my $guard = $minion->guard('processing_pkg_1', 172800); + ok !$minion->lock('processing_pkg_1', 0), 'lock exists'; + $worker->register; + ok my $job = $worker->dequeue(0, {id => $job_id}), 'job dequeued'; + is $job->execute, undef, 'no error'; + like $minion->job($job_id)->info->{result}, qr/Package \d+ is already being processed/, + 'race condition prevented'; + $worker->unregister; + undef $guard; + ok $minion->lock('processing_pkg_1', 0), 'lock no longer exists'; + }; + }; +}; + +done_testing; diff --git a/t/util.t b/t/util.t index ff96399ba2..00b455edd7 100644 --- a/t/util.t +++ b/t/util.t @@ -16,10 +16,10 @@ use Mojo::Base -strict; use Test::More; -use Mojo::File qw(curfile tempfile); +use Mojo::File qw(path curfile tempfile); use Mojo::JSON qw(decode_json); -use Cavil::Util - qw(buckets lines_context obs_ssh_auth parse_exclude_file pattern_matches request_id_from_external_link ssh_sign); +use Cavil::Util (qw(buckets lines_context obs_ssh_auth parse_exclude_file pattern_matches), + qw(request_id_from_external_link run_cmd ssh_sign)); my $PRIVATE_KEY = tempfile->spew(<<'EOF'); -----BEGIN OPENSSH PRIVATE KEY----- @@ -88,6 +88,15 @@ subtest 'request_id_from_external_link' => sub { is request_id_from_external_link(''), undef, 'no id'; }; +subtest 'run_cmd' => sub { + my $cwd = path; + my $result = run_cmd($cwd, ['echo', 'foo']); + is $result->{status}, !!1, 'right status'; + is $result->{exit_code}, 0, 'right exit code'; + is $result->{stderr}, '', 'right stderr'; + is $result->{stdout}, "foo\n", 'right stdout'; +}; + subtest 'ssh_sign' => sub { my $signature = ssh_sign($PRIVATE_KEY, 'realm', 'message'); like $signature, qr/^[-A-Za-z0-9+\/]+={0,3}$/, 'valid Base64 encoded signature';