-
-
Notifications
You must be signed in to change notification settings - Fork 469
/
project.yml
66 lines (60 loc) · 3.02 KB
/
project.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
title: "Ray integration"
description: "Use [Ray](https://ray.io) and the [`spacy-ray`](https://github.com/explosion/spacy-ray) extension package for parallel and distributed training. To configure the number of workers, you can change the `n_workers` variable in the `project.yml`."
# Variables can be referenced across the project.yml using ${vars.var_name}
vars:
config: "default"
treebank: "UD_English-EWT"
train_name: "en_ewt-ud-train"
dev_name: "en_ewt-ud-dev"
test_name: "en_ewt-ud-test"
n_workers: 2
# These are the directories that the project needs. The project CLI will make
# sure that they always exist.
directories: ["scripts", "corpus", "assets", "training", "configs"]
# Assets that should be downloaded or available in the directory. You can replace
# this with your own input data.
assets:
- dest: "assets/${vars.treebank}"
git:
repo: "https://github.com/UniversalDependencies/${vars.treebank}"
branch: "master"
path: ""
workflows:
all:
- convert
- train
# Project commands, specified in a style similar to CI config files (e.g. Azure
# pipelines). The name is the command name that lets you trigger the command
# via "spacy project run [command] [path]". The help message is optional and
# shown when executing "spacy project run [optional command] [path] --help".
commands:
- name: convert
help: "Convert the data to spaCy's format"
# Make sure we specify the branch in the command string, so that the
# caching works correctly.
script:
- "mkdir -p corpus/${vars.treebank}"
- "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"
- "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"
- "python -m spacy convert assets/${vars.treebank}/${vars.test_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"
- "mv corpus/${vars.treebank}/${vars.train_name}.spacy corpus/${vars.treebank}/train.spacy"
- "mv corpus/${vars.treebank}/${vars.dev_name}.spacy corpus/${vars.treebank}/dev.spacy"
- "mv corpus/${vars.treebank}/${vars.test_name}.spacy corpus/${vars.treebank}/test.spacy"
deps:
- "assets/${vars.treebank}/${vars.train_name}.conllu"
- "assets/${vars.treebank}/${vars.dev_name}.conllu"
- "assets/${vars.treebank}/${vars.test_name}.conllu"
outputs:
- "corpus/${vars.treebank}/train.spacy"
- "corpus/${vars.treebank}/dev.spacy"
- "corpus/${vars.treebank}/test.spacy"
- name: train
help: "Train ${vars.treebank}"
script:
- "python -m spacy ray train configs/${vars.config}.cfg --output training/${vars.treebank} --paths.train corpus/${vars.treebank}/train.spacy --paths.dev corpus/${vars.treebank}/dev.spacy --n-workers ${vars.n_workers}"
deps:
- "corpus/${vars.treebank}/train.spacy"
- "corpus/${vars.treebank}/dev.spacy"
- "configs/${vars.config}.cfg"
outputs:
- "training/${vars.treebank}/model-best"