From 2bd13881b52d3d9ec4ef1ad9e0b14d3165a8f7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20Ioni=C8=9B=C4=83?= Date: Fri, 8 Nov 2024 17:31:52 +0000 Subject: [PATCH] wip --- app/Actions/CheckRecordHasIssues.php | 46 ++++++++++ app/Actions/CheckVotable.php | 25 ++++++ app/Actions/GenerateMappedVotablesList.php | 60 +++++++++++++ app/Enums/Part.php | 28 +++++++ app/Enums/Time.php | 15 ++++ .../Imports/SimpleCandidateImporter.php | 23 +---- .../FetchRecordsJob.php} | 16 ++-- .../ImportAbroadRecordsJob.php} | 36 ++------ .../ImportCountyRecordsJob.php} | 79 +++++++++-------- .../Turnouts/ImportAbroadTurnoutsJob.php | 8 +- .../Turnouts/ImportCountyTurnoutsJob.php | 5 +- app/Livewire/Pages/ElectionPage.php | 2 +- app/Models/Candidate.php | 2 +- app/Models/Record.php | 2 +- app/Models/Vote.php | 63 ++++++++++++++ app/Policies/VotePolicy.php | 67 +++++++++++++++ app/Providers/AppServiceProvider.php | 1 + config/import.php | 1 + database/factories/VoteFactory.php | 25 ++++++ .../0001_01_10_000020_create_votes_table.php | 84 +++++++++++++++++++ 20 files changed, 482 insertions(+), 106 deletions(-) create mode 100644 app/Actions/CheckRecordHasIssues.php create mode 100644 app/Actions/CheckVotable.php create mode 100644 app/Actions/GenerateMappedVotablesList.php create mode 100644 app/Enums/Part.php create mode 100644 app/Enums/Time.php rename app/Jobs/Europarl240609/{Results/FetchResultsJob.php => Records/FetchRecordsJob.php} (75%) rename app/Jobs/Europarl240609/{Results/ImportAbroadResultsJob.php => Records/ImportAbroadRecordsJob.php} (79%) rename app/Jobs/Europarl240609/{Results/ImportCountyResultsJob.php => Records/ImportCountyRecordsJob.php} (52%) create mode 100644 app/Models/Vote.php create mode 100644 app/Policies/VotePolicy.php create mode 100644 database/factories/VoteFactory.php create mode 100644 database/migrations/0001_01_10_000020_create_votes_table.php diff --git a/app/Actions/CheckRecordHasIssues.php b/app/Actions/CheckRecordHasIssues.php new file mode 100644 index 0000000..d0d8732 --- /dev/null +++ b/app/Actions/CheckRecordHasIssues.php @@ -0,0 +1,46 @@ +map(fn (string $key) => $record[$key]) + ->sum(); + + if ($computedTotal !== $record['LT']) { + return true; + } + + return false; + } + + public function checkRecord(array $record): bool + { + if ($record['a'] != $record['a1'] + $record['a2']) { + return true; + } + + if ($record['a1'] < $record['b1']) { + return true; + } + + if ($record['a2'] < $record['b2']) { + return true; + } + + if ($record['b'] != $record['b1'] + $record['b2'] + $record['b3']) { + return true; + } + + if ($record['c'] < $record['d'] + $record['e'] + $record['f']) { + return true; + } + + return false; + } +} diff --git a/app/Actions/CheckVotable.php b/app/Actions/CheckVotable.php new file mode 100644 index 0000000..b3b28fa --- /dev/null +++ b/app/Actions/CheckVotable.php @@ -0,0 +1,25 @@ +getIndependentCandidatePrefix()); + } + + public function getName(string $name): string + { + return Str::afterLast($name, $this->getIndependentCandidatePrefix()); + } +} diff --git a/app/Actions/GenerateMappedVotablesList.php b/app/Actions/GenerateMappedVotablesList.php new file mode 100644 index 0000000..e14f8e8 --- /dev/null +++ b/app/Actions/GenerateMappedVotablesList.php @@ -0,0 +1,60 @@ + collect($header) + ->filter(fn (string $column) => $this->hasVotesSuffix($column)) + ->mapWithKeys(fn (string $column) => [ + $column => $this->getCandidateOrParty($column), + ]) + ); + } + + protected function getVotesSuffix(): string + { + return config('import.candidate_votes_suffix'); + } + + protected function hasVotesSuffix(string $column): bool + { + return Str::endsWith($column, $this->getVotesSuffix()); + } + + protected function getCandidateOrParty(string $name): array + { + $name = Str::before($name, $this->getVotesSuffix()); + + $votable = Party::query() + ->where('name', $name) + ->firstOr( + fn () => Candidate::query() + ->where('name', $name) + ->first() + ); + + if (blank($votable)) { + throw new \Exception("Votable not found for column: {$name}"); + } + + return [ + 'votable_type' => $votable?->getMorphClass(), + 'votable_id' => $votable?->id, + ]; + } +} diff --git a/app/Enums/Part.php b/app/Enums/Part.php new file mode 100644 index 0000000..f46361a --- /dev/null +++ b/app/Enums/Part.php @@ -0,0 +1,28 @@ + __('app.part.prov'), + self::PART => __('app.part.part'), + self::FINAL => __('app.part.final'), + }; + } +} diff --git a/app/Enums/Time.php b/app/Enums/Time.php new file mode 100644 index 0000000..1c9203d --- /dev/null +++ b/app/Enums/Time.php @@ -0,0 +1,15 @@ +isIndependentCandidate() + static::$model = app(CheckVotable::class)->isIndependentCandidate($this->data['name']) ? Candidate::class : Party::class; @@ -43,26 +43,9 @@ public function resolveRecord(): Candidate|Party ]); } - private function getIndependentCandidatePrefix(): string - { - return config('import.independent_candidate_prefix'); - } - - private function getIndependentCandidateName(): string - { - return Str::after($this->data['name'], $this->getIndependentCandidatePrefix()); - } - - private function isIndependentCandidate(): bool - { - return Str::startsWith($this->data['name'], $this->getIndependentCandidatePrefix()); - } - protected function afterValidate(): void { - if ($this->isIndependentCandidate()) { - $this->data['name'] = $this->getIndependentCandidateName(); - } + $this->data['name'] = app(CheckVotable::class)->getName($this->data['name']); } public static function getCompletedNotificationBody(Import $import): string diff --git a/app/Jobs/Europarl240609/Results/FetchResultsJob.php b/app/Jobs/Europarl240609/Records/FetchRecordsJob.php similarity index 75% rename from app/Jobs/Europarl240609/Results/FetchResultsJob.php rename to app/Jobs/Europarl240609/Records/FetchRecordsJob.php index ca57c25..e7afda6 100644 --- a/app/Jobs/Europarl240609/Results/FetchResultsJob.php +++ b/app/Jobs/Europarl240609/Records/FetchRecordsJob.php @@ -2,20 +2,21 @@ declare(strict_types=1); -namespace App\Jobs\Europarl240609\Results; +namespace App\Jobs\Europarl240609\Records; use App\Jobs\DeleteTemporaryTableData; use App\Jobs\PersistTemporaryTableData; use App\Jobs\SchedulableJob; use App\Models\County; use App\Models\Record; +use App\Models\Vote; use Illuminate\Support\Facades\Bus; -class FetchResultsJob extends SchedulableJob +class FetchRecordsJob extends SchedulableJob { public static function name(): string { - return 'Europarlamentare 09.06.2024 / Rezultate'; + return 'Europarlamentare 09.06.2024 / Procese Verbale'; } public function execute(): void @@ -28,12 +29,15 @@ public function execute(): void $time = now()->toDateTimeString(); $jobs = $counties - ->map(fn (County $county) => new ImportCountyResultsJob($this->scheduledJob, $county)) - ->push(new ImportAbroadResultsJob($this->scheduledJob)); + ->map(fn (County $county) => new ImportCountyRecordsJob($this->scheduledJob, $county)) + ->push(new ImportAbroadRecordsJob($this->scheduledJob)); $persistAndClean = fn () => Bus::chain([ new PersistTemporaryTableData(Record::class, $electionId), new DeleteTemporaryTableData(Record::class, $electionId), + + new PersistTemporaryTableData(Vote::class, $electionId), + new DeleteTemporaryTableData(Vote::class, $electionId), ])->dispatch(); Bus::batch($jobs) @@ -53,7 +57,7 @@ public function tags(): array { return [ 'import', - 'results', + 'records', 'scheduled_job:' . $this->scheduledJob->id, 'election:' . $this->scheduledJob->election_id, static::name(), diff --git a/app/Jobs/Europarl240609/Results/ImportAbroadResultsJob.php b/app/Jobs/Europarl240609/Records/ImportAbroadRecordsJob.php similarity index 79% rename from app/Jobs/Europarl240609/Results/ImportAbroadResultsJob.php rename to app/Jobs/Europarl240609/Records/ImportAbroadRecordsJob.php index 86e3b60..1044c2f 100644 --- a/app/Jobs/Europarl240609/Results/ImportAbroadResultsJob.php +++ b/app/Jobs/Europarl240609/Records/ImportAbroadRecordsJob.php @@ -2,8 +2,9 @@ declare(strict_types=1); -namespace App\Jobs\Europarl240609\Results; +namespace App\Jobs\Europarl240609\Records; +use App\Actions\CheckRecordHasIssues; use App\Events\CountryCodeNotFound; use App\Exceptions\CountryCodeNotFoundException; use App\Exceptions\MissingSourceFileException; @@ -18,7 +19,7 @@ use Illuminate\Queue\SerializesModels; use League\Csv\Reader; -class ImportAbroadResultsJob implements ShouldQueue +class ImportAbroadRecordsJob implements ShouldQueue { use Batchable; use Dispatchable; @@ -43,7 +44,7 @@ public function __construct(ScheduledJob $scheduledJob) $this->scheduledJob = $scheduledJob; } - public function handle(): void + public function handle(CheckRecordHasIssues $checker): void { $disk = $this->scheduledJob->disk(); $path = $this->scheduledJob->getSourcePath('sr.csv'); @@ -84,7 +85,7 @@ public function handle(): void 'votes_valid' => $record['e'], 'votes_null' => $record['f'], - 'has_issues' => $this->determineIfHasIssues($record), + 'has_issues' => $checker->checkRecord($record), ]); } catch (CountryCodeNotFoundException $th) { CountryCodeNotFound::dispatch($record['uat_name'], $this->scheduledJob->election); @@ -94,31 +95,6 @@ public function handle(): void Record::saveToTemporaryTable($values->all()); } - protected function determineIfHasIssues(array $record): bool - { - if ($record['a'] != $record['a1'] + $record['a2']) { - return true; - } - - if ($record['a1'] < $record['b1']) { - return true; - } - - if ($record['a2'] < $record['b2']) { - return true; - } - - if ($record['b'] != $record['b1'] + $record['b2'] + $record['b3']) { - return true; - } - - if ($record['c'] < $record['d'] + $record['e'] + $record['f']) { - return true; - } - - return false; - } - protected function getCountryId(string $name): string { $country = Country::search($name)->first(); @@ -139,7 +115,7 @@ public function tags(): array { return [ 'import', - 'results', + 'records', 'scheduled_job:' . $this->scheduledJob->id, 'election:' . $this->scheduledJob->election_id, 'abroad', diff --git a/app/Jobs/Europarl240609/Results/ImportCountyResultsJob.php b/app/Jobs/Europarl240609/Records/ImportCountyRecordsJob.php similarity index 52% rename from app/Jobs/Europarl240609/Results/ImportCountyResultsJob.php rename to app/Jobs/Europarl240609/Records/ImportCountyRecordsJob.php index 2ca8aa7..332d5c9 100644 --- a/app/Jobs/Europarl240609/Results/ImportCountyResultsJob.php +++ b/app/Jobs/Europarl240609/Records/ImportCountyRecordsJob.php @@ -2,12 +2,15 @@ declare(strict_types=1); -namespace App\Jobs\Europarl240609\Results; +namespace App\Jobs\Europarl240609\Records; +use App\Actions\CheckRecordHasIssues; +use App\Actions\GenerateMappedVotablesList; use App\Exceptions\MissingSourceFileException; use App\Models\County; use App\Models\Record; use App\Models\ScheduledJob; +use App\Models\Vote; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; @@ -17,7 +20,7 @@ use Illuminate\Support\Str; use League\Csv\Reader; -class ImportCountyResultsJob implements ShouldQueue +class ImportCountyRecordsJob implements ShouldQueue { use Batchable; use Dispatchable; @@ -45,7 +48,7 @@ public function __construct(ScheduledJob $scheduledJob, County $county) $this->county = $county; } - public function handle(): void + public function handle(CheckRecordHasIssues $checker, GenerateMappedVotablesList $generator): void { $disk = $this->scheduledJob->disk(); $path = $this->scheduledJob->getSourcePath("{$this->county->code}.csv"); @@ -64,58 +67,52 @@ public function handle(): void $reader = Reader::createFromStream($disk->readStream($path)); $reader->setHeaderOffset(0); - $values = collect(); + $records = collect(); - $records = $reader->getRecords(); - foreach ($records as $record) { - $values->push([ + $votables = $generator->votables($reader->getHeader()); + + foreach ($reader->getRecords() as $row) { + $records->push([ 'election_id' => $this->scheduledJob->election_id, 'county_id' => $this->county->id, - 'locality_id' => $record['uat_siruta'], - 'section' => $record['precinct_nr'], + 'locality_id' => $row['uat_siruta'], + 'section' => $row['precinct_nr'], - 'eligible_voters_permanent' => $record['a1'], - 'eligible_voters_special' => $record['a2'], + 'eligible_voters_permanent' => $row['a1'], + 'eligible_voters_special' => $row['a2'], - 'present_voters_permanent' => $record['b1'], - 'present_voters_special' => $record['b2'], - 'present_voters_supliment' => $record['b3'], + 'present_voters_permanent' => $row['b1'], + 'present_voters_special' => $row['b2'], + 'present_voters_supliment' => $row['b3'], - 'papers_received' => $record['c'], - 'papers_unused' => $record['d'], - 'votes_valid' => $record['e'], - 'votes_null' => $record['f'], + 'papers_received' => $row['c'], + 'papers_unused' => $row['d'], + 'votes_valid' => $row['e'], + 'votes_null' => $row['f'], - 'has_issues' => $this->determineIfHasIssues($record), + 'has_issues' => $checker->checkRecord($row), ]); - } - Record::saveToTemporaryTable($values->all()); - } + $votes = collect(); - protected function determineIfHasIssues(array $record): bool - { - if ($record['a'] != $record['a1'] + $record['a2']) { - return true; - } + foreach ($votables as $column => $votable) { + $votes->push([ + 'election_id' => $this->scheduledJob->election_id, + 'county_id' => $this->county->id, + 'locality_id' => $row['uat_siruta'], + 'section' => $row['precinct_nr'], - if ($record['a1'] < $record['b1']) { - return true; - } - - if ($record['a2'] < $record['b2']) { - return true; - } + 'votable_type' => $votable['votable_type'], + 'votable_id' => $votable['votable_id'], - if ($record['b'] != $record['b1'] + $record['b2'] + $record['b3']) { - return true; - } + 'votes' => $row[$column], + ]); + } - if ($record['c'] < $record['d'] + $record['e'] + $record['f']) { - return true; + Vote::saveToTemporaryTable($votes->all()); } - return false; + Record::saveToTemporaryTable($records->all()); } /** @@ -127,7 +124,7 @@ public function tags(): array { return [ 'import', - 'results', + 'records', 'scheduled_job:' . $this->scheduledJob->id, 'election:' . $this->scheduledJob->election_id, 'county:' . $this->county->code, diff --git a/app/Jobs/Europarl240609/Turnouts/ImportAbroadTurnoutsJob.php b/app/Jobs/Europarl240609/Turnouts/ImportAbroadTurnoutsJob.php index bc5b158..80c6ab7 100644 --- a/app/Jobs/Europarl240609/Turnouts/ImportAbroadTurnoutsJob.php +++ b/app/Jobs/Europarl240609/Turnouts/ImportAbroadTurnoutsJob.php @@ -4,6 +4,7 @@ namespace App\Jobs\Europarl240609\Turnouts; +use App\Actions\CheckRecordHasIssues; use App\Events\CountryCodeNotFound; use App\Exceptions\CountryCodeNotFoundException; use App\Exceptions\MissingSourceFileException; @@ -33,7 +34,7 @@ public function __construct(ScheduledJob $scheduledJob) $this->scheduledJob = $scheduledJob; } - public function handle(): void + public function handle(CheckRecordHasIssues $checker): void { $disk = $this->scheduledJob->disk(); $path = $this->scheduledJob->getSourcePath('SR.csv'); @@ -49,8 +50,7 @@ public function handle(): void $segments = Turnout::segmentsMap(); - $records = $reader->getRecords(); - foreach ($records as $record) { + foreach ($reader->getRecords() as $record) { try { $values->push([ 'election_id' => $this->scheduledJob->election_id, @@ -65,7 +65,7 @@ public function handle(): void 'mobile' => $record['UM'], 'area' => $record['Mediu'], - 'has_issues' => $this->determineIfHasIssues($record), + 'has_issues' => $checker->checkTurnout($record), ...$segments->map(fn (string $segment) => $record[$segment]), ]); diff --git a/app/Jobs/Europarl240609/Turnouts/ImportCountyTurnoutsJob.php b/app/Jobs/Europarl240609/Turnouts/ImportCountyTurnoutsJob.php index 3806c74..796f794 100644 --- a/app/Jobs/Europarl240609/Turnouts/ImportCountyTurnoutsJob.php +++ b/app/Jobs/Europarl240609/Turnouts/ImportCountyTurnoutsJob.php @@ -4,6 +4,7 @@ namespace App\Jobs\Europarl240609\Turnouts; +use App\Actions\CheckRecordHasIssues; use App\Exceptions\MissingSourceFileException; use App\Models\County; use App\Models\ScheduledJob; @@ -34,7 +35,7 @@ public function __construct(ScheduledJob $scheduledJob, County $county) $this->county = $county; } - public function handle(): void + public function handle(CheckRecordHasIssues $checker): void { $disk = $this->scheduledJob->disk(); $path = $this->scheduledJob->getSourcePath("{$this->county->code}.csv"); @@ -66,7 +67,7 @@ public function handle(): void 'mobile' => $record['UM'], 'area' => $record['Mediu'], - 'has_issues' => $this->determineIfHasIssues($record), + 'has_issues' => $checker->checkTurnout($record), ...$segments->map(fn (string $segment) => $record[$segment]), ]); diff --git a/app/Livewire/Pages/ElectionPage.php b/app/Livewire/Pages/ElectionPage.php index 4323f1d..7e160df 100644 --- a/app/Livewire/Pages/ElectionPage.php +++ b/app/Livewire/Pages/ElectionPage.php @@ -112,6 +112,6 @@ public function form(Form $form): Form #[Computed] public function mapKey(): string { - return md5("map-{$this->level->value}-{$this->county}"); + return hash('xxh128', "map-{$this->level->value}-{$this->county}"); } } diff --git a/app/Models/Candidate.php b/app/Models/Candidate.php index fab30d5..5119ef5 100644 --- a/app/Models/Candidate.php +++ b/app/Models/Candidate.php @@ -17,7 +17,7 @@ class Candidate extends Model implements HasMedia { use BelongsToElection; - /** @use HasFactory<\Database\Factories\CandidateFactory> */ + /** @use HasFactory */ use HasFactory; use InteractsWithMedia; diff --git a/app/Models/Record.php b/app/Models/Record.php index 1cb8237..4a6856a 100644 --- a/app/Models/Record.php +++ b/app/Models/Record.php @@ -15,7 +15,7 @@ class Record extends Model implements TemporaryTable { use BelongsToElection; - /** @use HasFactory<\Database\Factories\RecordFactory> */ + /** @use HasFactory */ use HasFactory; use HasTemporaryTable; diff --git a/app/Models/Vote.php b/app/Models/Vote.php new file mode 100644 index 0000000..25e6c23 --- /dev/null +++ b/app/Models/Vote.php @@ -0,0 +1,63 @@ + */ + use HasFactory; + use HasTemporaryTable; + + protected static string $factory = VoteFactory::class; + + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'election_id', + 'county_id', + 'locality_id', + 'section', + // 'part', + 'votable_type', + 'votable_id', + 'votes', + ]; + + /** + * Get the attributes that should be cast. + * + * @return array + */ + protected function casts(): array + { + return [ + // 'part' => Part::class, + 'votes' => 'integer', + ]; + } + + public function votable(): MorphTo + { + return $this->morphTo(); + } + + public function getTemporaryTableUniqueColumns(): array + { + return ['election_id', 'county_id', 'country_id', 'section', 'votable_type', 'votable_id']; + } +} diff --git a/app/Policies/VotePolicy.php b/app/Policies/VotePolicy.php new file mode 100644 index 0000000..5b270d3 --- /dev/null +++ b/app/Policies/VotePolicy.php @@ -0,0 +1,67 @@ + \App\Models\Record::class, 'turnout' => \App\Models\Turnout::class, 'user' => \App\Models\User::class, + 'vote' => \App\Models\Vote::class, ]); } diff --git a/config/import.php b/config/import.php index 3872403..2ddc7dd 100644 --- a/config/import.php +++ b/config/import.php @@ -9,5 +9,6 @@ 'disk' => env('IMPORT_DISK', 'local'), 'independent_candidate_prefix' => env('IMPORT_INDEPENDENT_CANDIDATE_PREFIX', 'CANDIDAT INDEPENDENT - '), + 'candidate_votes_suffix' => env('IMPORT_CANDIDATE_VOTES_SUFFIX', '-voturi'), ]; diff --git a/database/factories/VoteFactory.php b/database/factories/VoteFactory.php new file mode 100644 index 0000000..484f62e --- /dev/null +++ b/database/factories/VoteFactory.php @@ -0,0 +1,25 @@ + + */ +class VoteFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + // + ]; + } +} diff --git a/database/migrations/0001_01_10_000020_create_votes_table.php b/database/migrations/0001_01_10_000020_create_votes_table.php new file mode 100644 index 0000000..d404316 --- /dev/null +++ b/database/migrations/0001_01_10_000020_create_votes_table.php @@ -0,0 +1,84 @@ +id(); + + $table->foreignIdFor(Election::class) + ->constrained() + ->cascadeOnDelete(); + + $table->foreignIdFor(Country::class) + ->nullable() + ->constrained() + ->cascadeOnDelete(); + + $table->smallInteger('county_id') + ->unsigned() + ->nullable(); + + $table->foreign('county_id') + ->references('id') + ->on('counties'); + + $table->mediumInteger('locality_id') + ->unsigned() + ->nullable(); + + $table->foreign('locality_id') + ->references('id') + ->on('localities'); + + $table->string('section'); + $table->tinyInteger('part')->unsigned(); + + $table->morphs('votable'); + + $table->mediumInteger('votes')->unsigned()->default(0); + + $table->unique(['election_id', 'county_id', 'section', 'votable_type', 'votable_id'], 'votes_unique_county_index'); + $table->unique(['election_id', 'country_id', 'section', 'votable_type', 'votable_id'], 'votes_unique_country_index'); + }); + + Schema::create('_temp_votes', function (Blueprint $table) { + $table->bigInteger('election_id') + ->unsigned() + ->nullable(); + + $table->string('country_id', 2) + ->nullable(); + + $table->smallInteger('county_id') + ->unsigned() + ->nullable(); + + $table->mediumInteger('locality_id') + ->unsigned() + ->nullable(); + + $table->string('section'); + + $table->tinyInteger('part')->unsigned(); + + $table->string('votable_type'); + $table->bigInteger('votable_id') + ->unsigned(); + + $table->mediumInteger('votes')->unsigned(); + }); + } +};