-
Notifications
You must be signed in to change notification settings - Fork 48
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Conflicts: bin/metric.pl bin/restore.pl demo/demo.sh demo/demo2.pl install/determine_os.sh install/install_perl.sh lib/Scot.pm lib/Scot/App/Flair.pm lib/Scot/App/Game.pm lib/Scot/Collection.pm lib/Scot/Collection/Alertgroup.pm lib/Scot/Collection/Apikey.pm lib/Scot/Collection/Atmetric.pm lib/Scot/Collection/Checklist.pm lib/Scot/Collection/Entity.pm lib/Scot/Collection/Entry.pm lib/Scot/Collection/Event.pm lib/Scot/Collection/File.pm lib/Scot/Collection/Guide.pm lib/Scot/Collection/Handler.pm lib/Scot/Collection/Incident.pm lib/Scot/Collection/Intel.pm lib/Scot/Collection/Link.pm lib/Scot/Collection/Signature.pm lib/Scot/Collection/Tag.pm lib/Scot/Collection/User.pm lib/Scot/Controller/Api.pm lib/Scot/Controller/Auth.pm lib/Scot/Controller/Metric.pm lib/Scot/Model/Alertgroup.pm lib/Scot/Model/Signature.pm lib/Scot/Parser/Splunk.pm lib/Scot/Role/TLP.pm pubdev/gulpfile.js pubdev/jsdev/react_components/components/add_entry.jsx pubdev/jsdev/react_components/components/detail_data_status.jsx pubdev/jsdev/react_components/components/file_upload.jsx pubdev/jsdev/react_components/components/signature_table.jsx pubdev/jsdev/react_components/detail/selected_entry.jsx pubdev/jsdev/react_components/detail/selected_header.jsx pubdev/jsdev/react_components/detail/selected_header_options.jsx pubdev/jsdev/react_components/list/list-view.jsx pubdev/jsdev/react_components/main/index.jsx pubdev/jsdev/react_components/modal/entity_detail.jsx pubdev/jsdev/react_components/modal/link_warning.jsx pubdev/node_modules/prop-types/node_modules/fbjs/CHANGELOG.md pubdev/node_modules/prop-types/node_modules/fbjs/lib/Deferred.js pubdev/node_modules/prop-types/node_modules/fbjs/lib/Deferred.js.flow pubdev/node_modules/prop-types/node_modules/fbjs/lib/EventListener.js pubdev/node_modules/prop-types/node_modules/fbjs/lib/EventListener.js.flow pubdev/node_modules/prop-types/node_modules/fbjs/lib/warning.js pubdev/node_modules/prop-types/node_modules/fbjs/package.json pubdev/node_modules/prop-types/package.json pubdev/package.json pubdev/sass/components/esearch.scss public/css/listview.css public/css/sandbox.css public/css/styles.css public/css/styles.less public/index.html public/js/list-columns.js public/revl.html public/scot-3.5.js t/api/alertgroup.t t/api/entry.t t/api/event.t t/api/link.t t/api/promote.t t/entityextractor/email.t
- Loading branch information
Showing
28,174 changed files
with
33,085 additions
and
2,130,882 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| #!/usr/bin/env perl | ||
| use lib '../../Scot-Internal-Modules/lib'; | ||
| use lib '../lib'; | ||
| use strict; | ||
| use warnings; | ||
| use v5.18; | ||
|
|
||
| use Test::More; | ||
| use Test::Mojo; | ||
| use Data::Dumper; | ||
| use Mojo::JSON qw(encode_json decode_json); | ||
| use Scot::Env; | ||
| use HTML::Entities; | ||
|
|
||
| $ENV{'scot_mode'} = "testing"; | ||
| $ENV{'scot_auth_type'} = "Testing"; | ||
| $ENV{'scot_logfile'} = "/var/log/scot/scot.link.test.log"; | ||
| $ENV{'scot_config_file'} = '../../Scot-Internal-Modules/etc/scot.cfg.pl'; | ||
|
|
||
| my $env = Scot::Env->new({ | ||
| config_file => $ENV{'scot_config_file'}, | ||
| }); | ||
| my $mongo = $env->mongo; | ||
| my @colnames = (qw(tag source)); | ||
|
|
||
| foreach my $colnombre (@colnames) { | ||
|
|
||
| say "Deduplicating $colnombre"; | ||
|
|
||
| my %seen = (); | ||
|
|
||
| my $col = $mongo->collection(ucfirst($colnombre)); | ||
| my $cur = $col->find(); | ||
|
|
||
| ITEM: | ||
| while ( my $item = $cur->next ) { | ||
|
|
||
| my $id = $item->id; | ||
| my $value = $item->value; | ||
| my $lcval = lc($value); | ||
|
|
||
| say " $value : $id"; | ||
|
|
||
| if ( $value eq $lcval ) { | ||
| # special case, already lc'ed | ||
| say " ... already lowercased ... skipping" | ||
| $seen{$lcval}++; | ||
| next ITEM; | ||
| } | ||
|
|
||
|
|
||
| } | ||
| } | ||
|
|
||
| sub move_links { | ||
| my $tagsrc = shift; | ||
| my $oldobj = $tagsrc; | ||
| my $newobj = shift; | ||
|
|
||
| my $lcol = $mongo->collection('Link'); | ||
| my $lcur = $lcol->get_object_links($tagsrc); | ||
|
|
||
| while ( my $link = $lcur->next ) { | ||
| my $pullvert = $lcol->get_vertex($oldobj); | ||
| my $addvert = $lcol->get_vertex($newobj); | ||
| say " ... removing old link vertice ".$pullvert->{type}.":".$pullvert->{id}; | ||
| $link->update({'$pull' => { vertices => $pullvert }}); | ||
| say " ... adding new link vertice ".$addvert->{type}.":".$addvert->{id}; | ||
| $link->update({'$addToSet' => { vertices => $addvert }}); | ||
| } | ||
| } | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| #!/usr/bin/env perl | ||
|
|
||
| use MongoDB; | ||
| use Data::Dumper; | ||
| use v5.18; | ||
|
|
||
| my $mongo = MongoDB->connect->db('scot-prod'); | ||
| my $collection = $mongo->get_collection('link'); | ||
| my $cursor = $collection->find(); | ||
| $cursor->sort({id => -1}); | ||
| $cursor->limit(150000); | ||
| my %seen = (); | ||
| my $duplicates = 0; | ||
|
|
||
| LINK: | ||
| while (my $link = $cursor->next) { | ||
| my $v0 = $link->{vertices}->[0]; | ||
| my $v1 = $link->{vertices}->[1]; | ||
|
|
||
| my $m0 = join('-',$v0->{type},$v0->{id},$v1->{type},$v1->{id}); | ||
| my $m1 = join('-',$v1->{type},$v1->{id},$v0->{type},$v0->{id}); | ||
| my $id = $link->{id}; | ||
|
|
||
| if ( defined $seen{$m0} ) { | ||
| say "Link $id is a duplicate of ".$seen{$m0}; | ||
| $duplicates++; | ||
| $collection->delete_one({id => $id}); | ||
| next LINK; | ||
| } | ||
| if ( defined $seen{$m1} ) { | ||
| say "Link $id is a duplicate of ".$seen{$m1}." (inv)"; | ||
| $duplicates++; | ||
| $collection->delete_one({id => $id}); | ||
| next LINK; | ||
| } | ||
| say "Link $id is first occurrence of $m0"; | ||
| $seen{$m0} = $id; | ||
| } | ||
|
|
||
| say "$duplicates Duplicates found"; | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| #!/usr/bin/env perl | ||
|
|
||
| use MongoDB; | ||
| use Data::Dumper; | ||
| use v5.18; | ||
|
|
||
| #my $env = Scot::Env->new({ | ||
| # config_file => "../../Scot-Internal-Modules/etc/elu.cfg.pl", | ||
| #}); | ||
|
|
||
| my $mongo = MongoDB->connect->db('scot-prod'); | ||
| my $entitycol = $mongo->get_collection('entity'); | ||
| my $linkcol = $mongo->get_collection('link'); | ||
| my $nlcol = $mongo->get_collection('link2'); | ||
|
|
||
|
|
||
| my $entitycur = $entitycol->find({}); | ||
| $entitycur->immortal(1); | ||
| my $entity_remain = $entitycur->count; | ||
| # $entitycur->sort({id=>1}); | ||
|
|
||
| print "___Processing $entity_remain Entities\n"; | ||
|
|
||
| my %duplicate_entities = (); | ||
| my %entity_lookup = (); | ||
| my @batch = (); | ||
| my $batch_count = 0; | ||
|
|
||
|
|
||
| ENTITY: | ||
| while ( my $entity = $entitycur->next ) { | ||
| my $entity_lookup_id = $entity_lookup{$entity->{value}}; | ||
|
|
||
| say "Entity ".$entity->{value}." (".$entity->{id}.")"; | ||
|
|
||
| if ( defined $entity_lookup_id ) { | ||
| # this entity has duplicate records | ||
| # replace the id in the entity record with the first id | ||
| my $dupid = $entity->{id}; | ||
| $entity->{id} = $entity_lookup_id; | ||
| push @{$duplicate_entities{$entity->{value}}{$entity_lookup_id}}, | ||
| $dupid; | ||
| print " is a duplicate of $entity_lookup_id\n"; | ||
| } | ||
| else { | ||
| # this is the first time working with this entity | ||
| $entity_lookup{$entity->{value}} = $entity->{id}; | ||
| } | ||
|
|
||
| my @links = get_entity_links($entity); | ||
|
|
||
| foreach my $link (@links) { | ||
| # inserts are costly, but insert_many allows you do multiple hundreds | ||
| # of records in the same time as one record. | ||
| my $record = create_link_record($entity,$link); | ||
| push @batch, $record; | ||
| $batch_count = scalar(@batch); | ||
| if ( $batch_count > 999 ) { | ||
| print " ! Writing new Link Records\n";; | ||
| $nlcol->insert_many(\@batch); | ||
| $batch_count = 0; | ||
| @batch = (); | ||
| } | ||
| } | ||
| $entity_remain--; | ||
| print "--- $entity_remain entities remain to be processed\n"; | ||
| } | ||
|
|
||
| # in case some records still in @batch | ||
| if ( scalar(@batch) > 0 ) { | ||
| print "!!!! Writing new Link Records !!!!!\n"; | ||
| $nlcol->insert_many(\@batch); | ||
| } | ||
|
|
||
| remove_duplicate_entities(\%duplicate_entities); | ||
|
|
||
| sub remove_duplicate_entities { | ||
| my $dups = shift; | ||
|
|
||
| foreach my $value (keys %$dups) { | ||
| foreach my $id (sort keys %{$dups->{$value}}) { | ||
| my $id_aref = $dups->{$value}->{$id}; | ||
| print "$value ($id) has duplicates: \n"; | ||
| foreach (@$id_aref) { | ||
| print " $_\n"; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| sub get_entity_links { | ||
| my $entity = shift; | ||
| my $value = $entity->{value}; | ||
|
|
||
| my $elcursor = $linkcol->find({ | ||
| value => $value, | ||
| }); | ||
| # $elcursor->sort({id=>1}); | ||
| my $link_count = $elcursor->count; | ||
|
|
||
| print " Orig: $link_count links, "; | ||
|
|
||
| my @targets = (); | ||
| my %seen = (); | ||
| LINK: | ||
| while ( my $link = $elcursor->next ) { | ||
| my $key = $link->{target}->{type}.$link->{target}->{id}; | ||
| $seen{$key}++; | ||
| if ( $seen{$key} > 1 ) { | ||
| next LINK; | ||
| } | ||
| push @targets, $link; | ||
| } | ||
|
|
||
| say " now ".scalar(@targets) . " links"; | ||
|
|
||
| return wantarray ? @targets : \@targets; | ||
| } | ||
|
|
||
| sub create_link_record { | ||
| my $entity = shift; | ||
| my $link = shift; | ||
|
|
||
| my $entity_id = $entity->{id}; | ||
| my $link_id = $link->{id}; | ||
| my $targetid = $link->{target}->{id}; | ||
| my $targettype = $link->{target}->{type}; | ||
| my $when = $link->{when}; | ||
|
|
||
| my $vertices = [ | ||
| { id => $entity_id, type => "entity" }, | ||
| { id => $targetid, type => $targettype }, | ||
| ]; | ||
| my $record = { | ||
| id => $link_id, | ||
| weight => 1, | ||
| vertices=> $vertices, | ||
| when => $when, | ||
| }; | ||
| return $record; | ||
| } | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| #!/usr/bin/env perl | ||
|
|
||
| use strict; | ||
| use warnings; | ||
| use v5.18; | ||
|
|
||
| use lib '../../Scot-Internal-Modules/lib'; | ||
| use lib '../lib'; | ||
| use lib '/opt/scot/lib'; | ||
| use Scot::App::Mail; | ||
| use Scot::Env; | ||
| use Data::Dumper; | ||
|
|
||
| say "--- Starting Mail Ingester ---"; | ||
|
|
||
| my $config_file = $ENV{'scot_config_file'} // | ||
| '/opt/scot/etc/scot.cfg.pl'; | ||
|
|
||
| my $env = Scot::Env->new( | ||
| config_file => $config_file, | ||
| ); | ||
|
|
||
| my $mongo = $env->mongo; | ||
| my $metcol = $mongo->collection('Metric'); | ||
| my $now = $env->now; | ||
| my $minutes = 10; | ||
| my $ago = $now - ($minutes * 60 ); | ||
| my $cursor = $metcol->find({ | ||
| metric => qr/healthcheck received/i, | ||
| epoch => { '$gt' => $ago }, | ||
| }); | ||
|
|
||
| my $senders = $env->mail_watch_senders // []; | ||
|
|
||
| if ( ! defined $senders ) { | ||
| die "You need to create an array ref in $config_file for mail_watch_senders"; | ||
| } | ||
|
|
||
| my %seen; | ||
| foreach my $sender (@$senders) { | ||
| $seen{$sender} = 0; | ||
| } | ||
|
|
||
| while (my $stat = $cursor->next) { | ||
| my $subject = $stat->metric; | ||
| my $system = ( split(/ /,$subject) )[0]; | ||
| $seen{$system}++; | ||
| } | ||
|
|
||
| foreach my $sender (@$senders) { | ||
| if ( $seen{$sender} == 0 ) { | ||
| $env->mq->send("scot", { | ||
| action => "wall", | ||
| data => { | ||
| message => $sender . "has not sent health check email in $minutes minutes", | ||
| who => "health check", | ||
| when => $now, | ||
| } | ||
| }); | ||
| } | ||
| } | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.