forked from UniversalDependencies/tools
-
Notifications
You must be signed in to change notification settings - Fork 0
/
check_files.pl
executable file
·1390 lines (1326 loc) · 61.6 KB
/
check_files.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env perl
# Checks files to be distributed as Universal Dependencies.
# Copyright © 2015, 2016, 2017, 2018 Dan Zeman <[email protected]>
# License: GNU GPL
use utf8;
use open ':utf8';
binmode(STDIN, ':utf8');
binmode(STDOUT, ':utf8');
binmode(STDERR, ':utf8');
sub usage
{
print STDERR ("Usage: tools/check_files.pl\n");
print STDERR (" Must be called from the folder where all UD repositories are cloned as subfolders.\n");
print STDERR (" Will produce a complete report for the next UD release.\n");
print STDERR ("\n");
print STDERR (" or: tools/check_files.pl UD_Ancient_Greek-PROIEL\n");
print STDERR (" Will just check files and metadata of one treebank, report errors and exit.\n");
}
use Getopt::Long;
use LWP::Simple;
# Dan's sorting library
use csort;
# If this script is called from the parent folder, how can it find the UD library?
use lib 'tools';
use udlib;
# Include reports on future repositories (not scheduled for the upcoming release)?
# (If there is no README file, we will include the repository in the report and complain about the missing README.)
my $include_future = 0;
# Pull the latest changes from Github and show git status of each repository?
my $pull = 0;
# Validate all CoNLL-U files and report invalid ones?
my $validate = 0;
# Recompute statistics of all treebanks and push them to Github?
my $recompute_stats = 0;
# Tag all repositories with the new release? (The $tag variable is either empty or it contains the tag.)
my $tag = ''; # example: 'r1.0'
# Number of the current release as it is found in README files. Repositories targeting a later release will not be included.
my $current_release = 2.7;
# Month and year when the next release is expected. We use it in the announcement.
my $next_release_expected = 'May 2021';
my $announcement_min_size = 'less than 1,000 tokens';
my $announcement_max_size = 'over 3 million tokens';
# Path to the previous release is needed to compare the number of sentences and words.
# zen:/net/data/universal-dependencies-1.2
# mekong:C:\Users\Dan\Documents\Lingvistika\Projekty\universal-dependencies\release-1.2
my $oldpath = '/net/data/universal-dependencies-2.6';
###!!! Also check the new_treebanks_by_release hash in check_metadata()!
GetOptions
(
'release' => \$current_release,
'next-expected' => \$next_release_expected,
'ann-min-size' => \$announcement_min_size,
'ann-max-size' => \$announcement_max_size,
'oldpath' => \$oldpath,
'future' => \$include_future,
'pull' => \$pull,
'validate' => \$validate,
'stats' => \$recompute_stats,
'tag=s' => \$tag
);
# We need a mapping from the English names of the languages (as they appear in folder names) to their ISO codes and families.
my $languages_from_yaml = udlib::get_language_hash();
# If there is one argument, we interpret it as a treebank name, check the files
# and metadata of that treebank, and exit. We should check the arguments after
# options were read, although we do not expect options if the script is called
# on one treebank.
if(scalar(@ARGV)==1)
{
my $folder = $ARGV[0];
$folder =~ s:/$::;
my $n_errors = 0;
my @errors;
# The name of the folder: 'UD_' + language name + optional treebank identifier.
# Example: UD_Ancient_Greek-PROIEL
my ($language, $treebank) = udlib::decompose_repo_name($folder);
if(defined($language))
{
if(exists($languages_from_yaml->{$language}))
{
$langcode = $languages_from_yaml->{$language}{lcode};
my $key = $langcode;
$key .= '_'.lc($treebank) if($treebank ne '');
my $prefix = $key.'-ud';
chdir($folder) or die("Cannot enter folder $folder");
my $files = get_files($folder, $prefix, '.');
# Check that the expected files are present and that there are no extra files.
check_files($folder, $prefix, $files, \@errors, \$n_errors);
# Read the README file. We need to know whether this repository is scheduled for the upcoming release.
my $metadata = udlib::read_readme('.');
if(!defined($metadata))
{
push(@errors, "$folder: cannot read the README file: $!\n");
$n_errors++;
}
# Check that all required metadata items are present in the README file.
check_metadata($folder, $metadata, $current_release, \@errors, \$n_errors);
chdir('..') or die("Cannot return to the upper folder");
# Check that the language-specific documentation has at least the index (summary) page.
check_documentation($folder, $langcode, \@errors, \$n_errors);
}
else
{
push(@errors, "Unknown language $language.\n");
$n_errors++;
}
}
else
{
push(@errors, "Cannot parse folder name $folder.\n");
$n_errors++;
}
if($n_errors>0)
{
print(join('', @errors));
print("*** FAILED ***\n");
}
else
{
# Output similar to the validator.
print("*** PASSED ***\n");
}
# Exit 0 is considered success by the operating system.
exit($n_errors);
}
# This script expects to be invoked in the folder in which all the UD_folders
# are placed.
opendir(DIR, '.') or die('Cannot read the contents of the working folder');
my @folders = sort(grep {-d $_ && m/^UD_[A-Z]/} (readdir(DIR)));
closedir(DIR);
my $validhash = get_validation_results();
my %valid = %{$validhash};
my $n_folders_with_data = 0;
my $n_errors = 0;
my %languages_with_data;
my %families_with_data;
my %licenses;
my %genres;
my %contributors;
my %contributions; # for each contributor find treebanks they contributed to
my %contacts;
my %stats;
my %nw; # number of words in train|dev|test|all; indexed by folder name
my @unknown_folders; # cannot parse folder name or unknown language
my @nongit_folders; # folder is not a git repository
my @empty_folders; # does not contain data
my @future_folders; # scheduled for a future release (and we did not ask to include future data in the report)
my @invalid_folders; # at least one .conllu file does not pass validation
my @released_folders;
my %tcode_to_treebank_name;
foreach my $folder (@folders)
{
# The name of the folder: 'UD_' + language name + optional treebank identifier.
# Example: UD_Ancient_Greek-PROIEL
my ($language, $treebank) = udlib::decompose_repo_name($folder);
my $langcode;
if(defined($language))
{
if(exists($languages_from_yaml->{$language}))
{
$langcode = $languages_from_yaml->{$language}{lcode};
my $key = $langcode;
$key .= '_'.lc($treebank) if($treebank ne '');
$tcode_to_treebank_name{$key} = join(' ', ($language, $treebank));
my $prefix = $key.'-ud';
chdir($folder) or die("Cannot enter folder $folder");
# Skip folders that are not Git repositories even if they otherwise look OK.
if(!-d '.git')
{
push(@nongit_folders, $folder);
chdir('..') or die("Cannot return to the upper folder");
next;
}
# This is a git repository with data.
# Make sure it is up-to-date.
if($pull)
{
print("git pull $folder\n");
system('git pull --tags');
print(`git status`);
}
# Skip folders that do not contain any data, i.e. CoNLL-U files.
my $files = get_files($folder, $prefix, '.');
my $n = scalar(@{$files->{conllu}});
if($n==0)
{
push(@empty_folders, $folder);
chdir('..') or die("Cannot return to the upper folder");
next;
}
if(!$valid{$folder})
{
push(@invalid_folders, $folder);
chdir('..') or die("Cannot return to the upper folder");
next;
}
# Check that the expected files are present and that there are no extra files.
my @errors;
if(!check_files($folder, $prefix, $files, \@errors, \$n_errors))
{
print(join('', @errors));
splice(@errors);
}
# Read the README file. We need to know whether this repository is scheduled for the upcoming release.
my $metadata = udlib::read_readme('.');
if(!defined($metadata))
{
print("$folder: cannot read the README file: $!\n");
$n_errors++;
}
if(exists($metadata->{firstrelease}) && $metadata->{firstrelease} <= $current_release)
{
$metadata->{release} = 1;
}
if(!$metadata->{release} && !$include_future)
{
push(@future_folders, $folder);
chdir('..') or die("Cannot return to the upper folder");
next;
}
#------------------------------------------------------------------
# End of skipping. If we are here, we have a versioned UD folder
# with valid data. We know that the folder is going to be released.
# Count it and check it for possible problems.
$n_folders_with_data++;
push(@released_folders, $folder);
$languages_with_data{$language}++;
my $family = $languages_from_yaml->{$language}{family};
if(defined($family))
{
$family =~ s/^IE/Indo-European/;
# Keep only the family, discard the genus if present.
$family =~ s/,.*//;
$families_with_data{$family}++;
}
# Check that all required metadata items are present in the README file.
if(!check_metadata($folder, $metadata, $current_release, \@errors, \$n_errors))
{
print(join('', @errors));
splice(@errors);
}
if(!-f 'LICENSE.txt') ###!!! We have already reported that the file does not exist but that was a function where README contents is not known and no generating attempt was made.
{
print("$folder: missing LICENSE.txt (README says license is '$metadata->{License}')\n");
generate_license($metadata->{License});
$n_errors++;
}
# Check the names and sizes of the data files.
my $nwtrain = 0;
my $nwdev = 0;
my $nwtest = 0;
# In general, every treebank should have at least the test data.
# If there are more data files, zero or one of each of the following is expected: train, dev.
# Exception 1: Czech PDT has four train files: train-c, train-l, train-m, train-v.
# Exception 2: German HDT has two train files: train-a, train-b.
# No other CoNLL-U files are expected.
# It is also expected that if there is dev, there is also train.
# And if there is train, it should be same size or larger (in words) than both dev and test.
if($folder eq 'UD_Czech-PDT')
{
# The data is split into four files because of the size limits.
my $stats = collect_statistics_about_ud_file("$prefix-train-c.conllu");
$nwtrain = $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-l.conllu");
$nwtrain += $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-m.conllu");
$nwtrain += $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-v.conllu");
$nwtrain += $stats->{nword};
}
elsif($folder eq 'UD_German-HDT')
{
# The data is split into four files because of the size limits.
my $stats = collect_statistics_about_ud_file("$prefix-train-a-1.conllu");
$nwtrain = $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-a-2.conllu");
$nwtrain += $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-b-1.conllu");
$nwtrain += $stats->{nword};
$stats = collect_statistics_about_ud_file("$prefix-train-b-2.conllu");
$nwtrain += $stats->{nword};
}
else # all other treebanks
{
if(-f "$prefix-train.conllu")
{
my $stats = collect_statistics_about_ud_file("$prefix-train.conllu");
$nwtrain = $stats->{nword};
# If required, check that the file is valid.
if($validate && !is_valid_conllu("$prefix-train.conllu", $key))
{
print("$folder: invalid file $prefix-train.conllu\n");
$n_errors++;
}
}
}
# Look for development data.
if(-f "$prefix-dev.conllu")
{
my $stats = collect_statistics_about_ud_file("$prefix-dev.conllu");
$nwdev = $stats->{nword};
# If required, check that the file is valid.
if($validate && !is_valid_conllu("$prefix-dev.conllu", $key))
{
print("$folder: invalid file $prefix-dev.conllu\n");
$n_errors++;
}
}
# Look for test data.
if(-f "$prefix-test.conllu")
{
my $stats = collect_statistics_about_ud_file("$prefix-test.conllu");
$nwtest = $stats->{nword};
# If required, check that the file is valid.
if($validate && !is_valid_conllu("$prefix-test.conllu", $key))
{
print("$folder: invalid file $prefix-test.conllu\n");
$n_errors++;
}
}
# Remember the numbers of words. We will need them for some tests
# that can only be done when all folders have been scanned.
my $nwall = $nwtrain+$nwdev+$nwtest;
$nw{$folder} = { 'train' => $nwtrain, 'dev' => $nwdev, 'test' => $nwtest, 'all' => $nwall };
# For small and growing treebanks, we expect the files to appear roughly in the following order:
# 1. test (>=10K tokens if possible);
# 2. train (if it can be larger than test or if this is the only treebank of the language and train is a small sample);
# 3. dev (if it can be at least 10K tokens and if train is larger than both test and dev).
if($nwtest==0 && ($nwtrain>0 || $nwdev>0))
{
print("$folder: train or dev exists but there is no test\n");
$n_errors++;
}
# Exception: PUD parallel data are currently test only, even if in some languages there is more than 20K words.
# Exception: UD_French-FQB is a test-only treebank (or use cross-validation, or add it to training data of Sequoia).
# Exception: UD_German-LIT is a test-only treebank (intended primarily for linguistic research).
# Exception: ParTUT has some portions smaller because of other limitations (sync across languages and with UD_Italian).
# Exception: TWITTIRO overlaps with POSTWITA and tries to match its data split.
# Exception: UD_Scottish_Gaelic-ARCOSG is close to 10K test tokens but they could not get there if they did not want to split documents.
if($nwall>10000 && $nwtest<10000 && $folder !~ m/^UD_(.+-ParTUT|Italian-TWITTIRO|Scottish_Gaelic-ARCOSG)$/)
{
print("$folder: more than 10K words (precisely: $nwall) available but test has only $nwtest words\n");
$n_errors++;
}
if($nwall>20000 && $nwtrain<10000 && $folder !~ m/^UD_(French-FQB|German-LIT|.+-PUD|.+-ParTUT)$/)
{
print("$folder: more than 20K words (precisely: $nwall) available but train has only $nwtrain words\n");
$n_errors++;
}
if($nwall>30000 && $nwdev<5000 && $folder !~ m/^UD_(German-LIT|.+-PUD|.+-ParTUT)$/)
{
print("$folder: more than 30K words (precisely: $nwall) available but dev has only $nwdev words\n");
$n_errors++;
}
$stats{$key} = collect_statistics_about_ud_treebank('.', $key);
# Summarize metadata.
if($metadata->{'License'} ne '')
{
$licenses{$metadata->{'License'}}++;
}
if($metadata->{'Genre'} ne '')
{
my @genres = split(/\s+/, $metadata->{'Genre'});
foreach my $genre (@genres)
{
$genres{$genre}++;
}
}
if($metadata->{'Contributors'} ne '')
{
my @contributors = split(/;\s*/, $metadata->{'Contributors'});
foreach my $contributor (@contributors)
{
$contributor =~ s/^\s+//;
$contributor =~ s/\s+$//;
$contributors{$contributor}++;
$contributions{$contributor}{$folder}++;
}
}
if($metadata->{'Contact'} ne '')
{
my @contacts = split(/[,;]\s*/, $metadata->{'Contact'});
foreach my $contact (@contacts)
{
$contact =~ s/^\s+//;
$contact =~ s/\s+$//;
$contacts{$contact}++;
}
}
# Recompute statistics of the treebank and push it back to Github.
if($recompute_stats)
{
print("Recomputing statistics of $folder...\n");
system('cat *.conllu | ../tools/conllu-stats.pl > stats.xml');
print("Pushing statistics to Github...\n");
system('git add stats.xml');
system('git commit -m "Updated statistics."');
if($tag ne '')
{
print("Tagging $folder $tag\n");
system("git tag $tag");
}
system('git push');
system('git push --tags');
}
chdir('..') or die("Cannot return to the upper folder");
}
else
{
print("Unknown language $language.\n");
push(@unknown_folders, $folder);
}
}
else
{
print("Cannot parse folder name $folder.\n");
push(@unknown_folders, $folder);
}
}
print("$n_errors errors must be fixed.\n") if($n_errors>0);
print("\n");
print("Found ", scalar(@folders), " folders.\n");
if(scalar(@unknown_folders) > 0)
{
print(scalar(@unknown_folders), " folders skipped because their language cannot be identified: ", join(', ', @unknown_folders), "\n");
}
if(scalar(@nongit_folders) > 0)
{
print(scalar(@nongit_folders), " folders ignored because they are not git repositories: ", join(', ', @nongit_folders), "\n");
}
if(scalar(@empty_folders) > 0)
{
print(scalar(@empty_folders), " folders ignored because they are empty: ", join(', ', @empty_folders), "\n");
}
if(scalar(@future_folders) > 0)
{
print(scalar(@future_folders), " folders ignored because their README says they should be released later: ", join(', ', @future_folders), "\n");
}
if(scalar(@invalid_folders) > 0)
{
print(scalar(@invalid_folders), " folders ignored because at least one file does not pass validation: ", join(', ', @invalid_folders), "\n");
}
# Do not separate names of released folders by commas. We will want to copy the list as arguments for the packaging script.
print("$n_folders_with_data folders are git repositories and contain valid data:\n\n", join(' ', @released_folders), "\n\n");
my @families = sort(keys(%families_with_data));
print(scalar(@families), " families with data: ", join(', ', @families), "\n\n");
my @languages = map {s/_/ /g; $_} (sort(keys(%languages_with_data)));
print(scalar(@languages), " languages with data: ", join(', ', @languages), "\n\n");
my @langcodes = sort(keys(%stats));
print("Treebank codes: ", join(' ', @langcodes), "\n\n");
my %langcodes1; map {my $x=$_; $x=~s/_.*//; $langcodes1{$x}++} (@langcodes);
my @langcodes1 = sort(keys(%langcodes1));
print("Language codes: ", join(' ', @langcodes1), "\n\n");
my @licenses = sort(keys(%licenses));
print(scalar(@licenses), " different licenses: ", join(', ', @licenses), "\n\n");
my @genres = sort(keys(%genres));
print(scalar(@genres), " different genres: ", join(', ', @genres), "\n\n");
my @contributors = keys(%contributors);
my %tridl, %tridf;
foreach my $contributor (@contributors)
{
# We want to sort by last names first, and only look at first names when the
# last names are identical. If we compared the whole names directly, we would
# see "Morioka, Tomohiko" between "Mori, Keiko" and "Mori, Shinsuke"; we do
# not want this to happen.
my $lastname = $contributor;
my $firstname = '';
if($contributor =~ m/^([^,]+),\s*(.+)$/)
{
$lastname = $1;
$firstname = $2;
}
$tridl{$contributor} = csort::zjistit_tridici_hodnoty($lastname, 'en');
$tridf{$contributor} = csort::zjistit_tridici_hodnoty($firstname, 'en');
}
# Since release 2.5 we go by "Zeman, Nivre, and alphabetically others".
# Normal trid values are numeric strings. Prepend '!' and it will sort before
# any numeric value.
$tridl{'Zeman, Daniel'} = '!01';
$tridl{'Nivre, Joakim'} = '!02';
@contributors = sort {my $r = $tridl{$a} cmp $tridl{$b}; unless($r) {$r = $tridf{$a} cmp $tridf{$b}} $r} (keys(%contributors));
# Is the same person spelled differently in different treebanks?
get_potentially_misspelled_contributors(\%contributions, @contributors);
my @contributors_firstlast = map {my $x = $_; if($x =~ m/^(.+?),\s*(.+)$/) {$x = "$2 $1";} $x} (@contributors);
print(scalar(@contributors), " contributors: ", join('; ', @contributors), "\n\n");
my @contacts = sort(keys(%contacts));
print(scalar(@contacts), " contacts: ", join(', ', @contacts), "\n\n");
# Find treebanks whose data size has changed.
print("Collecting statistics of $oldpath...\n");
my $stats11 = collect_statistics_about_ud_release($oldpath);
my @languages11 = sort(keys(%{$stats11}));
foreach my $l (@languages11)
{
print("$l\tt=$stats11->{$l}{ntok}\tw=$stats11->{$l}{nword}\tf=$stats11->{$l}{nfus}\ts=$stats11->{$l}{nsent}\n");
if($stats11->{$l}{ntok} != $stats{$l}{ntok} ||
$stats11->{$l}{nword} != $stats{$l}{nword} ||
$stats11->{$l}{nfus} != $stats{$l}{nfus} ||
$stats11->{$l}{nsent} != $stats{$l}{nsent})
{
print(" NOW:\tt=$stats{$l}{ntok}\tw=$stats{$l}{nword}\tf=$stats{$l}{nfus}\ts=$stats{$l}{nsent}\n");
}
}
print("\n");
# Treebanks can appear, disappear and reappear. Get the list of all treebanks
# that are part either of this or of the previous release.
my %lastcurrtreebanks;
foreach my $t (@langcodes, @languages11)
{
$lastcurrtreebanks{$t}++;
}
# Find treebanks whose size has changed by more than 10%.
my @changedsize;
my $codemaxl = 0;
my $namemaxl = 0;
my $oldmaxl = 0;
my $newmaxl = 0;
foreach my $t (sort(keys(%lastcurrtreebanks)))
{
my $oldsize = exists($stats11->{$t}) ? $stats11->{$t}{nword} : 0;
my $newsize = exists($stats{$t}) ? $stats{$t}{nword} : 0;
if($newsize > $oldsize * 1.1 || $newsize < $oldsize * 0.9)
{
my %record =
(
'code' => $t,
'name' => $tcode_to_treebank_name{$t},
'old' => $oldsize,
'new' => $newsize
);
push(@changedsize, \%record);
$codemaxl = length($t) if(length($t) > $codemaxl);
$namemaxl = length($record{name}) if(length($record{name}) > $namemaxl);
$oldmaxl = length($oldsize) if(length($oldsize) > $oldmaxl);
$newmaxl = length($newsize) if(length($newsize) > $newmaxl);
}
}
my $nchangedsize = scalar(@changedsize);
my $changelog = "The size of the following $nchangedsize treebanks changed significantly since the last release:\n";
foreach my $r (sort {$a->{name} cmp $b->{name}} (@changedsize))
{
my $padding = ' ' x ($namemaxl - length($r->{name}));
$changelog .= sprintf(" %s: %${oldmaxl}d → %${newmaxl}d\n", $r->{name}.$padding, $r->{old}, $r->{new}); # right arrow is \x{2192}
}
# Collect statistics of the current treebanks. Especially the total number of
# sentences, tokens and words is needed for the metadata in Lindat.
my $ntok = 0;
my $nword = 0;
my $nfus = 0;
my $nsent = 0;
foreach my $l (@langcodes)
{
print("$l\tt=$stats{$l}{ntok}\tw=$stats{$l}{nword}\tf=$stats{$l}{nfus}\ts=$stats{$l}{nsent}\n");
$ntok += $stats{$l}{ntok};
$nword += $stats{$l}{nword};
$nfus += $stats{$l}{nfus};
$nsent += $stats{$l}{nsent};
}
print("TOTAL\tt=$ntok\tw=$nword\tf=$nfus\ts=$nsent\n");
print("--------------------------------------------------------------------------------\n");
my $announcement = get_announcement
(
$current_release,
$n_folders_with_data,
\@languages,
\@families,
$announcement_min_size,
$announcement_max_size,
$next_release_expected,
\@contributors_firstlast,
$changelog
);
print($announcement);
#------------------------------------------------------------------------------
# Downloads the current validation report from the validation server. Returns
# a reference to the hash of valid treebanks.
#------------------------------------------------------------------------------
sub get_validation_results
{
my %valid;
# After we used this script to select the treebanks automatically,
# we typically freeze the list in an external file called
# released_treebanks.txt (see http://universaldependencies.org/release_checklist_task_force.html#determining-which-treebanks-will-be-released).
# Download the current validation report. (We could run the validator ourselves
# but it would take a lot of time.)
my @validation_report = split(/\n/, get('https://quest.ms.mff.cuni.cz/udvalidator/cgi-bin/unidep/validation-report.pl?text_only'));
if(scalar(@validation_report)==0)
{
print STDERR ("WARNING: Could not download validation report from quest. All treebanks will be considered invalid.\n");
}
foreach my $line (@validation_report)
{
if($line =~ m/^(UD_.+): (VALID|LEGACY)/)
{
$valid{$1}++;
}
}
return \%valid;
}
#------------------------------------------------------------------------------
# Gets the list of files in a UD folder. Returns the list of CoNLL-U files, and
# the list of unexpected extra files.
#------------------------------------------------------------------------------
sub get_files
{
my $folder = shift; # name of the UD repository
my $prefix = shift; # prefix of data files, i.e., language code _ treebank code
my $path = shift; # path to the repository (default: '.')
$path = '.' if(!defined($path));
opendir(DIR, $path) or die("Cannot read the contents of the folder $folder");
my @files = readdir(DIR);
closedir(DIR);
my @conllufiles = grep {-f $_ && m/\.conllu$/} (@files);
# Look for additional files so they can be reported.
# Some extra files are tolerated in the Github repository although we do not include them in the release package; these are not reported.
my @tolerated =
(
# tolerated but not released
'\.\.?',
'\.git(ignore|attributes)?',
'\.travis\.yml',
'not-to-release',
# expected and released
'README\.(txt|md)',
'LICENSE\.txt',
'CONTRIBUTING\.md',
$prefix.'-(train|dev|test)\.conllu',
'stats\.xml',
# split data files of large treebanks
'cs_pdt-ud-train-[clmv]\.conllu',
'de_hdt-ud-train-[ab]-[12]\.conllu'
);
my $tolerated_re = join('|', @tolerated);
my @extrafiles = map
{
$_ .= '/' if(-d $_);
$_
}
grep
{
!m/^($tolerated_re)$/
}
(@files);
# Some treebanks have exceptional extra files that have been approved and released previously.
# The treebanks without underlying text need a program that merges the CoNLL-U files with the separately distributed text.
# Learner corpora need extra columns to annotate "distributional tag", "distributional head", "distributional relation" and "alignment".
###!!! But we now have guidelines for extra columns from the PARSEME project, so there will be a CoNLL-U-Plus format, .conllup.
@extrafiles = grep
{!(
$folder eq 'UD_Arabic-NYUAD' && $_ eq 'merge.jar' ||
$folder eq 'UD_Chinese-CFL' && $_ eq 'zh_cfl-ud-test.conllux' ||
$folder eq 'UD_English-ESL' && $_ eq 'merge.py' ||
$folder eq 'UD_English-GUMReddit' && $_ eq 'get_text.py' ||
$folder eq 'UD_Hindi_English-HIENCS' && $_ =~ m/^(merge\/?|crawl_tweets\.py)$/ ||
$folder eq 'UD_Japanese-KTC' && $_ =~ m/^merge/ ||
$folder eq 'UD_Japanese-BCCWJ' && $_ =~ m/^merge/
)}
(@extrafiles);
my %files =
(
'conllu' => \@conllufiles,
'extra' => \@extrafiles
);
return \%files;
}
#------------------------------------------------------------------------------
# Checks whether a UD repository contains the expected files. Expects that the
# repository is our current folder.
#------------------------------------------------------------------------------
sub check_files
{
my $folder = shift; # folder name, e.g. 'UD_Czech-PDT', not path
my $prefix = shift; # prefix of names of data files, e.g. 'cs_pdt-ud'
my $files = shift; # hash of files in the folder as collected by get_files()
my $errors = shift; # reference to array of error messages
my $n_errors = shift; # reference to error counter
my $ok = 1;
if(!-f 'README.txt' && !-f 'README.md')
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing README.txt|md\n");
$$n_errors++;
}
if(-f 'README.txt' && -f 'README.md')
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: both README.txt and README.md are present\n");
$$n_errors++;
}
if(!-f 'LICENSE.txt')
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing LICENSE.txt\n");
$$n_errors++;
}
# Check the data files.
my $train_found = 0;
# In general, every treebank should have at least the test data.
# If there are more data files, zero or one of each of the following is expected: train, dev.
# Exception 1: Czech PDT has four train files: train-c, train-l, train-m, train-v.
# Exception 2: German HDT has two train files: train-a, train-b.
# No other CoNLL-U files are expected.
# It is also expected that if there is dev, there is also train.
if($folder eq 'UD_Czech-PDT')
{
# The data is split into four files because of the size limits.
if(!-f "$prefix-train-c.conllu" || !-f "$prefix-train-l.conllu" || !-f "$prefix-train-m.conllu" || !-f "$prefix-train-v.conllu")
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing at least one file of $prefix-train-[clmv].conllu\n");
$$n_errors++;
}
else
{
$train_found = 1;
}
}
elsif($folder eq 'UD_German-HDT')
{
# The data is split into four files because of the size limits.
if(!-f "$prefix-train-a-1.conllu" || !-f "$prefix-train-a-2.conllu" || !-f "$prefix-train-b-1.conllu" || !-f "$prefix-train-b-2.conllu")
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing at least one file of $prefix-train-[ab]-[12].conllu\n");
$$n_errors++;
}
else
{
$train_found = 1;
}
}
else # all other treebanks
{
if(-f "$prefix-train.conllu")
{
# Not finding train is not automatically an error. The treebank can be test-only.
$train_found = 1;
}
}
# Look for development data. They are optional and not finding them is not an error.
if(-f "$prefix-dev.conllu")
{
# However, if there is dev data, there should also be training data!
if(!$train_found)
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing training data although there is dev data\n");
$$n_errors++;
}
}
# Look for test data. Unlike train and dev, test data is mandatory!
if(!-f "$prefix-test.conllu")
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder: missing test data file $prefix-test.conllu\n");
$$n_errors++;
}
# Extra files have already been identified but not registered as an error.
if(scalar(@{$files->{extra}})>0)
{
$ok = 0;
push(@{$errors}, "[L0 Repo files] $folder extra files: ".join(', ', sort(@{$files->{extra}}))."\n");
$$n_errors += scalar(@{$files->{extra}});
}
return $ok;
}
#------------------------------------------------------------------------------
# Checks whether metadata in the README file provides required information.
#------------------------------------------------------------------------------
sub check_metadata
{
my $folder = shift; # folder name, e.g. 'UD_Czech-PDT', not path
my $metadata = shift; # reference to hash returned by udlib::read_readme()
my $current_release = shift; # needed to know whether changelog is required
my $errors = shift; # reference to array of error messages
my $n_errors = shift; # reference to error counter
my $ok = 1;
# New contributors sometimes forget to add it. Old contributors sometimes modify it for no good reason ('Data available since' should never change!)
# And occasionally people delete the metadata section completely, despite being told not to do so (Hebrew team in the last minute of UD 2.0!)
if($metadata->{'Data available since'} =~ m/UD\s*v([0-9]+\.[0-9]+)/)
{
my $claimed = $1;
# The value 'Data available since' must not change from release to release.
# It must forever refer to the first release of the treebank in UD.
# Therefore, this script will remember the correct value, too, and shout if it changes in the README.
my %new_treebanks_by_release =
(
'1.0' => ['Czech-PDT', 'English-EWT', 'Finnish-TDT', 'French-GSD', 'German-GSD', 'Hungarian-Szeged', 'Irish-IDT', 'Italian-ISDT', 'Spanish-GSD', 'Swedish-Talbanken'],
'1.1' => ['Basque-BDT', 'Bulgarian-BTB', 'Croatian-SET', 'Danish-DDT', 'Finnish-FTB', 'Greek-GDT', 'Hebrew-HTB', 'Indonesian-GSD', 'Persian-Seraji'],
'1.2' => ['Ancient_Greek-Perseus', 'Ancient_Greek-PROIEL', 'Arabic-PADT', 'Dutch-Alpino', 'Estonian-EDT', 'Gothic-PROIEL', 'Hindi-HDTB', 'Japanese-KTC', 'Latin-ITTB', 'Latin-Perseus', 'Latin-PROIEL', 'Norwegian-Bokmaal', 'Old_Church_Slavonic-PROIEL', 'Polish-PDB', 'Portuguese-Bosque', 'Romanian-RRT', 'Slovenian-SSJ', 'Tamil-TTB'],
'1.3' => ['Catalan-AnCora', 'Czech-CAC', 'Czech-CLTT', 'Dutch-LassySmall', 'English-ESL', 'English-LinES', 'Galician-CTG', 'Chinese-GSD', 'Kazakh-KTB', 'Latvian-LVTB', 'Portuguese-GSD', 'Russian-GSD', 'Russian-SynTagRus', 'Slovenian-SST', 'Spanish-AnCora', 'Swedish-LinES', 'Turkish-IMST'],
'1.4' => ['Coptic-Scriptorium', 'Galician-TreeGal', 'Japanese-GSD', 'Sanskrit-UFAL', 'Slovak-SNK', 'Swedish_Sign_Language-SSLC', 'Ukrainian-IU', 'Uyghur-UDT', 'Vietnamese-VTB'],
'2.0' => ['Arabic-NYUAD', 'Belarusian-HSE', 'English-ParTUT', 'French-FTB', 'French-ParTUT', 'French-Sequoia', 'Italian-ParTUT', 'Korean-GSD', 'Lithuanian-HSE', 'Norwegian-Nynorsk', 'Urdu-UDTB'],
'2.1' => ['Afrikaans-AfriBooms', 'Arabic-PUD', 'Buryat-BDT', 'Cantonese-HK', 'Czech-FicTree', 'Czech-PUD', 'English-PUD', 'Finnish-PUD', 'French-PUD', 'German-PUD', 'Hindi-PUD', 'Chinese-CFL', 'Chinese-HK', 'Chinese-PUD', 'Italian-PoSTWITA', 'Italian-PUD', 'Japanese-PUD', 'Kurmanji-MG', 'Marathi-UFAL', 'North_Sami-Giella', 'Norwegian-NynorskLIA', 'Portuguese-PUD', 'Romanian-Nonstandard', 'Russian-PUD', 'Serbian-SET', 'Spanish-PUD', 'Swedish-PUD', 'Telugu-MTG', 'Turkish-PUD', 'Upper_Sorbian-UFAL'],
'2.2' => ['Amharic-ATT', 'Armenian-ArmTDP', 'Breton-KEB', 'English-GUM', 'Faroese-OFT', 'French-Spoken', 'Indonesian-PUD', 'Japanese-BCCWJ', 'Japanese-Modern', 'Komi_Zyrian-IKDP', 'Komi_Zyrian-Lattice', 'Korean-Kaist', 'Korean-PUD', 'Naija-NSC', 'Old_French-SRCMF', 'Polish-LFG', 'Russian-Taiga', 'Tagalog-TRG', 'Thai-PUD', 'Warlpiri-UFAL', 'Yoruba-YTB'],
'2.3' => ['Akkadian-PISANDUB', 'Bambara-CRB', 'Erzya-JR', 'Hindi_English-HIENCS', 'Maltese-MUDT'],
'2.4' => ['Assyrian-AS', 'Classical_Chinese-Kyoto', 'Estonian-EWT', 'French-FQB', 'German-HDT', 'German-LIT', 'Italian-VIT', 'Karelian-KKPP', 'Lithuanian-ALKSNIS', 'Mbya_Guarani-Dooley', 'Mbya_Guarani-Thomas', 'Old_Russian-RNC', 'Old_Russian-TOROT', 'Polish-PUD', 'Turkish-GB', 'Welsh-CCG', 'Wolof-WTB'],
'2.5' => ['Bhojpuri-BHTB', 'Chinese-GSDSimp', 'English-Pronouns', 'Italian-TWITTIRO', 'Komi_Permyak-UH', 'Livvi-KKPP', 'Moksha-JR', 'Romanian-SiMoNERo', 'Scottish_Gaelic-ARCOSG', 'Skolt_Sami-Giellagas', 'Swiss_German-UZH'],
'2.6' => ['Albanian-TSA', 'English-GUMReddit', 'Icelandic-PUD', 'Latin-LLCT', 'Sanskrit-Vedic', 'Tagalog-Ugnayan'],
'2.7' => ['Akkadian-RIAO', 'Akuntsu-TuDeT', 'Apurina-UFPA', 'Chukchi-HSE', 'Faroese-FarPaHC', 'Finnish-OOD', 'Icelandic-IcePaHC', 'Indonesian-CSUI', 'Khunsari-AHA', 'Manx-Cadhan', 'Munduruku-TuDeT', 'Nayini-AHA', 'Old_Turkish-Tonqq', 'Persian-PerDT', 'Soi-AHA', 'South_Levantine_Arabic-MADAR', 'Tamil-MWTT', 'Tupinamba-TuDeT', 'Turkish-BOUN', 'Turkish_German-SAGT']
);
my $correct;
foreach my $release (keys(%new_treebanks_by_release))
{
foreach my $treebank (@{$new_treebanks_by_release{$release}})
{
if("UD_$treebank" eq $folder)
{
$correct = $release;
last;
}
}
}
if(defined($correct) && $claimed ne $correct)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: 'Data available since: $claimed' is not true. This treebank was first released in UD v$correct.\n");
$$n_errors++;
}
elsif(!defined($correct) && $claimed < $current_release)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: 'Data available since: $claimed' is not true. This treebank was not released prior to UD v$current_release.\n");
$$n_errors++;
}
}
else
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Unknown format of Data available since: '$metadata->{'Data available since'}'\n");
$$n_errors++;
}
if($metadata->{Genre} !~ m/\w/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing list of genres: '$metadata->{Genre}'\n");
$$n_errors++;
}
else
{
# Originally (until UD 2.2) it was not an error if people invented their genres in addition to the predefined ones.
# However, some treebanks do not follow prescribed syntax (e.g. place commas between genres) or just have typos here
# (e.g. besides "news" there is also "new" or "newswire"), so we better ban unregistered genres and check it automatically.
# Note that a copy of the list of known genres is also in evaluate_treebank.pl and in docs-automation/genre_symbols.json.
my @official_genres = ('academic', 'bible', 'blog', 'email', 'fiction', 'government', 'grammar-examples', 'learner-essays', 'legal', 'medical', 'news', 'nonfiction', 'poetry', 'reviews', 'social', 'spoken', 'web', 'wiki');
my @genres = split(/\s+/, $metadata->{Genre});
my @unknown_genres = grep {my $g = $_; my @found = grep {$_ eq $g} (@official_genres); scalar(@found)==0} (@genres);
if(scalar(@unknown_genres)>0)
{
$ok = 0;
my $ug = join(' ', sort(@unknown_genres));
push(@{$errors}, "[L0 Repo readme] $folder README: Unknown genre '$ug'\n");
$$n_errors++;
}
}
if($metadata->{License} !~ m/\w/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing identification of license in README: '$metadata->{License}'\n");
$$n_errors++;
}
if($metadata->{'Includes text'} !~ m/^(yes|no)$/i)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Metadata 'Includes text' must be 'yes' or 'no' but the current value is: '$metadata->{'Includes text'}'\n");
$$n_errors++;
}
foreach my $annotation (qw(Lemmas UPOS XPOS Features Relations))
{
if($metadata->{$annotation} !~ m/\w/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing information on availability and source of $annotation\n");
$$n_errors++;
}
elsif($metadata->{$annotation} !~ m/^(manual native|converted from manual|converted with corrections|automatic|automatic with corrections|not available)$/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Unknown value of metadata $annotation: '$metadata->{$annotation}'\n");
$$n_errors++;
}
}
if($metadata->{Contributing} !~ m/\w/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing metadata Contributing (where and how to contribute)\n");
$$n_errors++;
}
elsif($metadata->{Contributing} !~ m/^(here|here source|elsewhere|to be adopted)$/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Unknown value of metadata Contributing: '$metadata->{Contributing}'\n");
$$n_errors++;
}
if($metadata->{Contributors} !~ m/\w/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing list of contributors: '$metadata->{Contributors}'\n");
$$n_errors++;
}
if($metadata->{Contact} !~ m/\@/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Missing contact e-mail: '$metadata->{Contact}'\n");
$$n_errors++;
}
# Check other sections of the README file.
if(!defined($metadata->{sections}{summary}))
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Section Summary not found.\n");
$$n_errors++;
}
elsif(length($metadata->{sections}{summary})<40)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Section Summary is too short.\n");
$$n_errors++;
}
elsif(length($metadata->{sections}{summary})>500)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Section Summary is too long.\n");
$$n_errors++;
}
elsif($metadata->{sections}{summary} =~ m/see \[release checklist\]/)
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Section Summary still contains the templatic text. Please put a real summary there.\n");
$$n_errors++;
}
if($metadata->{'Data available since'} =~ m/UD\s*v([0-9]+\.[0-9]+)/ && $1 < $current_release && !$metadata->{changelog})
{
$ok = 0;
push(@{$errors}, "[L0 Repo readme] $folder README: Old treebank ($metadata->{'Data available since'}) but README does not contain 'ChangeLog'\n");
$$n_errors++;
}
# Add a link to the guidelines for README files. Add it to the last error message.
# Do not make it a separate error message (just in case we get rid of $n_errors and use scalar(@errors) in the future).
unless($ok)
{
$errors->[-1] .= "See http://universaldependencies.org/release_checklist.html#treebank-metadata for guidelines on machine-readable metadata.\n";
$errors->[-1] .= "See http://universaldependencies.org/release_checklist.html#the-readme-file for general guidelines on README files.\n";
}
return $ok;
}
#------------------------------------------------------------------------------
# Checks whether documentation contains a summary page about a language.
#------------------------------------------------------------------------------
sub check_documentation
{
my $folder = shift; # folder name, e.g. 'UD_Czech-PDT', not path
my $lcode = shift;
my $errors = shift; # reference to array of error messages
my $n_errors = shift; # reference to error counter
my $ok = 1;
###!!! For now assume that a clone of the docs repository is accessible as
###!!! the docs subfolder of the current folder.
my $indexpath = "docs/_$lcode/index.md";
if(! -f $indexpath)
{
$ok = 0;
push(@{$errors}, "[L0 Repo lang-spec-doc] $folder: Language '$lcode' does not have the one-page documentation summary in the docs repository.\nSee http://universaldependencies.org/contributing_language_specific.html for instructions on how to write documentation.\n");
$$n_errors++;
}
else
{
# So the file exists but does it really contain anything useful?
# Some people just create an almost empty file without bothering to put the contents there (e.g., Sebastian for English).
my $doc;
open(IDX, $indexpath);
while(<IDX>)
{
$doc .= $_;
}
close(IDX);
# Czech documentation has over 16000 B.
# Swedish documentation has over 4500 B.
# Yoruba is probably incomplete but it still has over 3500 B.
# Let's require 2500 B as a minimum and hope that people don't just put a sequence of whitespace characters there.