From 813d73abaf1271a6fddf22556270549bd33ac408 Mon Sep 17 00:00:00 2001 From: hsolbrig Date: Wed, 18 Aug 2021 11:41:46 -0500 Subject: [PATCH] Update make and build files --- .github/workflows/main.yaml | 1 - .github/workflows/pr-test.yaml | 8 +- .gitignore | 6 +- Makefile | 175 ++++++++++++++++++++------------- Pipfile | 2 + environment.sh | 10 -- hide_test_changes.sh | 4 +- 7 files changed, 115 insertions(+), 91 deletions(-) delete mode 100755 environment.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6220c0f5..8f047bd2 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -26,7 +26,6 @@ jobs: - name: Run make file run: | - pip install pipx make - name: Run unit tests over output diff --git a/.github/workflows/pr-test.yaml b/.github/workflows/pr-test.yaml index f6f70da8..c6e6aebd 100644 --- a/.github/workflows/pr-test.yaml +++ b/.github/workflows/pr-test.yaml @@ -21,11 +21,11 @@ jobs: - name: Install pipenv uses: dschep/install-pipenv-action@v1 - - name: Install dependencies + - name: Run make file run: | - pipenv install --dev + make - - name: Run a full make and unit test + - name: Run unit tests over results run: | - make + pipenv install --dev pipenv run python -m unittest discover -p 'test_*.py' diff --git a/.gitignore b/.gitignore index 45b00782..c893d6f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,12 @@ # Elements that are built using Make +.venv/ target/ /docs/ +/make-venv/ + # Ignore all of linkml_model except the model directory linkml_model/ !linkml_model/model -pipx_bin/ -site/ -venv/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Makefile b/Makefile index 08963444..adc7ae0a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # ---------------------------------------- PKG_DIR = linkml_model SRC_DIR = $(PKG_DIR)/model -SCHEMA_DIR = $(PKG_DIR)/model/schema +SCHEMA_DIR = $(SRC_DIR)/schema MODEL_DOCS_DIR = $(SRC_DIR)/docs SOURCE_FILES := $(shell find $(SCHEMA_DIR) -name '*.yaml') SCHEMA_NAMES = $(patsubst $(SCHEMA_DIR)/%.yaml, %, $(SOURCE_FILES)) @@ -16,34 +16,54 @@ SCHEMA_SRC = $(SCHEMA_DIR)/$(SCHEMA_NAME).yaml PKG_TGTS = graphql json jsonld jsonschema owl rdf shex TGTS = docs python $(PKG_TGTS) -# Run a linkml package -RUN = +# Targets by PKG_TGT +PKG_T_GRAPHQL = $(PKG_DIR)/graphql +PKG_T_JSON = $(PKG_DIR)/json +PKG_T_JSONLD_CONTEXT = $(PKG_DIR)/jsonld +PKG_T_JSON_SCHEMA = $(PKG_DIR)/jsonschema +PKG_T_OWL = $(PKG_DIR)/owl +PKG_T_RDF = $(PKG_DIR)/rdf +PKG_T_SHEX = $(PKG_DIR)/shex +PKG_T_SQLDDL = $(PKG_DIR)/sqlddl +PKG_T_DOCS = $(MODEL_DOCS_DIR) +PKG_T_PYTHON = $(PKG_DIR) # Global generation options -GEN_OPTS = --log_level WARNING -im $(PKG_DIR)/model/import_map.json +GEN_OPTS = --log_level WARNING +ENV = export PIPENV_VENV_IN_PROJECT=true && export PIPENV_PIPFILE=make-venv/Pipfile && export PIPENV_IGNORE_VIRTUALENVS=1 +RUN = $(ENV) && pipenv run # ---------------------------------------- # TOP LEVEL TARGETS # ---------------------------------------- -all: install gen +all: install gen test # --------------------------------------- # We don't want to pollute the python environment with linkml tool specific packages. For this reason, -# we install an isolated instance of linkml +# we install an isolated instance of linkml in the pipenv-linkml directory # --------------------------------------- -install: .venv - . ./environment.sh -.PHONY: install +install: make-venv/env.lock + +make-venv/env.lock: + mkdir -p make-venv + touch make-venv/Pipfile + $(ENV) && pipenv install linkml mkdocs + touch make-venv/env.lock uninstall: - rm -rf .venv + touch make-venv/env.lock && rm -f make-venv/env.lock + if [ -f "make-venv/Pipfile" ]; then $(ENV) && pipenv --rm ; fi + rm -f make-venv/Pipfile* -.venv: - mkdir -p .venv && touch .venv/Pipfile - pipenv install "linkml~=1.0" - pipenv install mkdocs +# --------------------------------------- +# Test runner +# ---------------------------------------- +test: + pipenv install --dev + pipenv run python -m unittest + # --------------------------------------- # GEN: run generator for each target # --------------------------------------- @@ -59,13 +79,12 @@ clean: # --------------------------------------- # SQUEAKY_CLEAN: remove all of the final targets to make sure we don't leave old artifacts around # --------------------------------------- -squeaky-clean: clean $(patsubst %,squeaky-clean-%,$(PKG_TGTS)) +squeaky-clean: uninstall clean $(patsubst %,squeaky-clean-%,$(PKG_TGTS)) find docs/* ! -name 'README.*' -exec rm -rf {} + find $(PKG_DIR) -name "*.py" ! -name "__init__.py" ! -name "linkml_files.py" -exec rm -f {} + - rm -rf .venv squeaky-clean-%: clean - find $(PKG_DIR)/$* ! -name 'README.*' ! -name $* -type f -exec rm -f {} + + find $(PKG_DIR)/$* ! -name model ! -name 'README.*' ! -name $* -type f -exec rm -f {} + # --------------------------------------- # T: List files to generate @@ -79,142 +98,156 @@ t: echo: echo $(patsubst %,gen-%,$(TGTS)) - tdir-%: rm -rf target/$* mkdir -p target/$* -docs: - mkdir -p $@ - - # --------------------------------------- # MARKDOWN DOCS # Generate documentation ready for mkdocs # --------------------------------------- -gen-docs: docs/index.md +gen-docs: docs/index.html .PHONY: gen-docs -docs/index.md: target/docs/index.md +docs/index.html: target/docs/index.md install + echo "HERE!" $(PKG_T_DOCS) + mkdir -p $(PKG_T_DOCS) cp -R $(MODEL_DOCS_DIR)/*.md target/docs - $(RUN)mkdocs build + # mkdocs.yml moves from the target/docs to the docs directory + $(RUN) mkdocs build + target/docs/index.md: $(SCHEMA_DIR)/$(SCHEMA_NAME).yaml tdir-docs install - $(RUN)gen-markdown $(GEN_OPTS) --mergeimports --notypesdir --warnonexist --dir target/docs $< + $(RUN) gen-markdown $(GEN_OPTS) --mergeimports --notypesdir --warnonexist --dir target/docs $< -# --------------------------------------- -# YAML source -# --------------------------------------- -# YAML source is no longewr copied to the output directory. Instead, it is placed there via setup.cfg -gen-model: -.PHONY: gen-model # --------------------------------------- -# python source +# PYTHON Source # --------------------------------------- -gen-python: $(patsubst %, $(PKG_DIR)/%.py, $(SCHEMA_NAMES)) -$(PKG_DIR)/%.py: target/python/%.py +gen-python: $(patsubst %, $(PKG_T_PYTHON)/%.py, $(SCHEMA_NAMES)) +$(PKG_T_PYTHON)/%.py: target/python/%.py + mkdir -p $(PKG_T_PYTHON) cp $< $@ target/python/%.py: $(SCHEMA_DIR)/%.yaml tdir-python install - $(RUN)gen-python $(GEN_OPTS) --genmeta --no-slots --no-mergeimports $< > $@ + $(RUN) gen-python $(GEN_OPTS) --no-slots --no-mergeimports $< > $@ # --------------------------------------- # GRAPHQL Source # --------------------------------------- -# TODO: modularize imports. For now imports are merged. -gen-graphql: $(PKG_DIR)/graphql/$(SCHEMA_NAME).graphql +gen-graphql: $(PKG_T_GRAPHQL)/$(SCHEMA_NAME).graphql .PHONY: gen-graphql -$(PKG_DIR)/graphql/%.graphql: target/graphql/%.graphql +$(PKG_T_GRAPHQL)/%.graphql: target/graphql/%.graphql + mkdir -p $(PKG_T_GRAPHQL) cp $< $@ + target/graphql/%.graphql: $(SCHEMA_DIR)/%.yaml tdir-graphql install - $(RUN)gen-graphql $(GEN_OPTS) $< > $@ + $(RUN) gen-graphql $(GEN_OPTS) $< > $@ # --------------------------------------- # JSON Schema # --------------------------------------- -gen-jsonschema: $(patsubst %, $(PKG_DIR)/jsonschema/%.schema.json, $(SCHEMA_NAMES)) +gen-jsonschema: $(patsubst %, $(PKG_T_JSON_SCHEMA)/%.schema.json, $(SCHEMA_NAMES)) .PHONY: gen-jsonschema -$(PKG_DIR)/jsonschema/%.schema.json: target/jsonschema/%.schema.json + +$(PKG_T_JSON_SCHEMA)/%.schema.json: target/json_schema/%.schema.json + mkdir -p $(PKG_T_JSON_SCHEMA) cp $< $@ -target/jsonschema/%.schema.json: $(SCHEMA_DIR)/%.yaml tdir-jsonschema install - $(RUN)gen-json-schema $(GEN_OPTS) -t transaction $< > $@ + +target/json_schema/%.schema.json: $(SCHEMA_DIR)/%.yaml tdir-json_schema install + $(RUN) gen-json-schema $(GEN_OPTS) -t transaction $< > $@ # --------------------------------------- # ShEx # --------------------------------------- -gen-shex: $(patsubst %, $(PKG_DIR)/shex/%.shex, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/shex/%.shexj, $(SCHEMA_NAMES)) +gen-shex: $(patsubst %, $(PKG_T_SHEX)/%.shex, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_SHEX)/%.shexj, $(SCHEMA_NAMES)) .PHONY: gen-shex -$(PKG_DIR)/shex/%.shex: target/shex/%.shex +$(PKG_T_SHEX)/%.shex: target/shex/%.shex + mkdir -p $(PKG_T_SHEX) cp $< $@ -$(PKG_DIR)/shex/%.shexj: target/shex/%.shexj +$(PKG_T_SHEX)/%.shexj: target/shex/%.shexj + mkdir -p $(PKG_T_SHEX) cp $< $@ target/shex/%.shex: $(SCHEMA_DIR)/%.yaml tdir-shex install - $(RUN)gen-shex --no-mergeimports $(GEN_OPTS) $< > $@ + $(RUN) gen-shex --no-mergeimports $(GEN_OPTS) $< > $@ target/shex/%.shexj: $(SCHEMA_DIR)/%.yaml tdir-shex install - $(RUN)gen-shex --no-mergeimports $(GEN_OPTS) -f json $< > $@ + $(RUN) gen-shex --no-mergeimports $(GEN_OPTS) -f json $< > $@ # --------------------------------------- # OWL # --------------------------------------- -# TODO: modularize imports. For now imports are merged. -gen-owl: $(PKG_DIR)/owl/$(SCHEMA_NAME).owl.ttl +gen-owl: $(PKG_T_OWL)/$(SCHEMA_NAME).owl.ttl .PHONY: gen-owl -$(PKG_DIR)/owl/%.owl.ttl: target/owl/%.owl.ttl +$(PKG_T_OWL)/%.owl.ttl: target/owl/%.owl.ttl + mkdir -p $(PKG_T_OWL) cp $< $@ target/owl/%.owl.ttl: $(SCHEMA_DIR)/%.yaml tdir-owl install - $(RUN)gen-owl $(GEN_OPTS) $< > $@ + $(RUN) gen-owl $(GEN_OPTS) $< > $@ # --------------------------------------- # JSON-LD Context # --------------------------------------- -gen-jsonld: $(patsubst %, $(PKG_DIR)/jsonld/%.context.jsonld, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/jsonld/%.model.context.jsonld, $(SCHEMA_NAMES)) $(PKG_DIR)/jsonld/context.jsonld +gen-jsonld: $(patsubst %, $(PKG_T_JSONLD_CONTEXT)/%.context.jsonld, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_JSONLD_CONTEXT)/%.model.context.jsonld, $(SCHEMA_NAMES)) .PHONY: gen-jsonld -$(PKG_DIR)/jsonld/%.context.jsonld: target/jsonld/%.context.jsonld - cp $< $@ - -$(PKG_DIR)/jsonld/%.model.context.jsonld: target/jsonld/%.model.context.jsonld +$(PKG_T_JSONLD_CONTEXT)/%.context.jsonld: target/jsonld/%.context.jsonld + mkdir -p $(PKG_T_JSONLD_CONTEXT) cp $< $@ -$(PKG_DIR)/jsonld/context.jsonld: target/jsonld/meta.context.jsonld +$(PKG_T_JSONLD_CONTEXT)/%.model.context.jsonld: target/jsonld/%.model.context.jsonld + mkdir -p $(PKG_T_JSONLD_CONTEXT) cp $< $@ target/jsonld/%.context.jsonld: $(SCHEMA_DIR)/%.yaml tdir-jsonld install - $(RUN)gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@ + $(RUN) gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@ + target/jsonld/%.model.context.jsonld: $(SCHEMA_DIR)/%.yaml tdir-jsonld install - $(RUN)gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@ + $(RUN) gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@ # --------------------------------------- # Plain Old (PO) JSON # --------------------------------------- -gen-json: $(patsubst %, $(PKG_DIR)/json/%.json, $(SCHEMA_NAMES)) +gen-json: $(patsubst %, $(PKG_T_JSON)/%.json, $(SCHEMA_NAMES)) .PHONY: gen-json -$(PKG_DIR)/json/%.json: target/json/%.json +$(PKG_T_JSON)/%.json: target/json/%.json + mkdir -p $(PKG_T_JSON) cp $< $@ target/json/%.json: $(SCHEMA_DIR)/%.yaml tdir-json install - $(RUN)gen-jsonld $(GEN_OPTS) --no-mergeimports $< > $@ + $(RUN) gen-jsonld $(GEN_OPTS) --no-mergeimports $< > $@ # --------------------------------------- # RDF # --------------------------------------- -gen-rdf: gen-jsonld $(patsubst %, $(PKG_DIR)/rdf/%.ttl, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/rdf/%.model.ttl, $(SCHEMA_NAMES)) +gen-rdf: gen-jsonld $(patsubst %, $(PKG_T_RDF)/%.ttl, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_RDF)/%.model.ttl, $(SCHEMA_NAMES)) .PHONY: gen-rdf -$(PKG_DIR)/rdf/%.ttl: target/rdf/%.ttl +$(PKG_T_RDF)/%.ttl: target/rdf/%.ttl + mkdir -p $(PKG_T_RDF) cp $< $@ -$(PKG_DIR)/rdf/%.model.ttl: target/rdf/%.model.ttl +$(PKG_T_RDF)/%.model.ttl: target/rdf/%.model.ttl + mkdir -p $(PKG_T_RDF) cp $< $@ target/rdf/%.ttl: $(SCHEMA_DIR)/%.yaml $(PKG_DIR)/jsonld/%.context.jsonld tdir-rdf install - $(RUN)gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@ + $(RUN) gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@ target/rdf/%.model.ttl: $(SCHEMA_DIR)/%.yaml $(PKG_DIR)/jsonld/%.model.context.jsonld tdir-rdf install - $(RUN)gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@ + $(RUN) gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@ +# --------------------------------------- +# SQLDDL +# --------------------------------------- +gen-sqlddl: $(PKG_T_SQLDDL)/$(SCHEMA_NAME).sql +.PHONY: gen-sqlddl + +$(PKG_T_SQLDDL)/%.sql: target/sqlddl/%.sql + mkdir -p $(PKG_T_SQLDDL) + cp $< $@ +target/sqlddl/%.sql: $(SCHEMA_DIR)/%.yaml tdir-sqlddl install + $(RUN) gen-sqlddl $(GEN_OPTS) $< > $@ # test docs locally. -docserve: gen-docs install - $(RUN)mkdocs serve +docserve: gen-docs + $(RUN) mkdocs serve diff --git a/Pipfile b/Pipfile index 4d0f4673..8ead41c6 100644 --- a/Pipfile +++ b/Pipfile @@ -5,6 +5,8 @@ name = "pypi" [packages] linkml-runtime = "~=1.0, >=1.0.10" +linkml = "~=1.0" +mkdocs = "*" [dev-packages] linkml = "~=1.0, >=1.0.1" diff --git a/environment.sh b/environment.sh deleted file mode 100755 index 4e8b657a..00000000 --- a/environment.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -export PIPENV_VENV_IN_PROJECT=1 -export PATH=.venv/bin:$PATH -export PYTHONPATH=.venv:$PYTHONPATH -if [ "x" != x ] ; then - PS1="${PS1-}" -else - PS1="(environment: .venv) ${PS1-}" -fi -export PS1 diff --git a/hide_test_changes.sh b/hide_test_changes.sh index be4e389f..38cff03f 100755 --- a/hide_test_changes.sh +++ b/hide_test_changes.sh @@ -1,5 +1,5 @@ #!/bin/bash # Make all of the test output files invisible to git -echo git update-index --assume-unchanged `git status -s docs | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//'` -echo git update-index --assume-unchanged `git status -s linkml_model | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//' | grep -v \/model\/` +git update-index --assume-unchanged `git status -s docs | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//'` +git update-index --assume-unchanged `git status -s linkml_model | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//' | grep -v \/model\/`