Update make and build files

.github/workflows/main.yaml

@@ -26,7 +26,6 @@ jobs:
 
     - name: Run make file
       run: |
-        pip install pipx
         make
 
     - name: Run unit tests over output

.github/workflows/pr-test.yaml

@@ -21,11 +21,11 @@ jobs:
     - name: Install pipenv
       uses: dschep/install-pipenv-action@v1
 
-    - name: Install dependencies
+    - name: Run make file
       run: |
-       pipenv install --dev
+        make
 
-    - name: Run a full make and unit test
+    - name: Run unit tests over results
       run: |
-        make
+        pipenv install --dev
         pipenv run python -m unittest discover -p 'test_*.py'

.gitignore

@@ -1,12 +1,12 @@
 # Elements that are built using Make
+.venv/
 target/
 /docs/
+/make-venv/
+
 # Ignore all of linkml_model except the model directory
 linkml_model/
 !linkml_model/model
-pipx_bin/
-site/
-venv/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Makefile

@@ -6,7 +6,7 @@
 # ----------------------------------------
 PKG_DIR = linkml_model
 SRC_DIR = $(PKG_DIR)/model
-SCHEMA_DIR = $(PKG_DIR)/model/schema
+SCHEMA_DIR = $(SRC_DIR)/schema
 MODEL_DOCS_DIR = $(SRC_DIR)/docs
 SOURCE_FILES := $(shell find $(SCHEMA_DIR) -name '*.yaml')
 SCHEMA_NAMES = $(patsubst $(SCHEMA_DIR)/%.yaml, %, $(SOURCE_FILES))
@@ -16,34 +16,54 @@ SCHEMA_SRC = $(SCHEMA_DIR)/$(SCHEMA_NAME).yaml
 PKG_TGTS = graphql json jsonld jsonschema owl rdf shex
 TGTS = docs python $(PKG_TGTS)
 
-# Run a linkml package
-RUN =
+# Targets by PKG_TGT
+PKG_T_GRAPHQL = $(PKG_DIR)/graphql
+PKG_T_JSON = $(PKG_DIR)/json
+PKG_T_JSONLD_CONTEXT = $(PKG_DIR)/jsonld
+PKG_T_JSON_SCHEMA = $(PKG_DIR)/jsonschema
+PKG_T_OWL = $(PKG_DIR)/owl
+PKG_T_RDF = $(PKG_DIR)/rdf
+PKG_T_SHEX = $(PKG_DIR)/shex
+PKG_T_SQLDDL = $(PKG_DIR)/sqlddl
+PKG_T_DOCS = $(MODEL_DOCS_DIR)
+PKG_T_PYTHON = $(PKG_DIR)
 
 # Global generation options
-GEN_OPTS = --log_level WARNING -im $(PKG_DIR)/model/import_map.json
+GEN_OPTS = --log_level WARNING
+ENV = export PIPENV_VENV_IN_PROJECT=true && export PIPENV_PIPFILE=make-venv/Pipfile && export PIPENV_IGNORE_VIRTUALENVS=1
+RUN = $(ENV) && pipenv run
 
 # ----------------------------------------
 # TOP LEVEL TARGETS
 # ----------------------------------------
-all: install gen
+all: install gen test
 
 # ---------------------------------------
 # We don't want to pollute the python environment with linkml tool specific packages.  For this reason,
-# we install an isolated instance of linkml
+# we install an isolated instance of linkml in the pipenv-linkml directory
 # ---------------------------------------
-install: .venv
-	. ./environment.sh
-.PHONY: install
+install: make-venv/env.lock
+
+make-venv/env.lock:
+	mkdir -p make-venv
+	touch make-venv/Pipfile
+	$(ENV) && pipenv install linkml mkdocs
+	touch make-venv/env.lock
 
 uninstall:
-	rm -rf .venv
+	touch make-venv/env.lock && rm -f make-venv/env.lock
+	if [ -f "make-venv/Pipfile" ]; then $(ENV) && pipenv --rm ; fi
+	rm -f make-venv/Pipfile*
 
-.venv:
-	mkdir -p .venv && touch .venv/Pipfile
-	pipenv install "linkml~=1.0"
-	pipenv install mkdocs
 
 
+# ---------------------------------------
+# Test runner
+# ----------------------------------------
+test:
+	pipenv install --dev
+	pipenv run python -m unittest
+
 # ---------------------------------------
 # GEN: run generator for each target
 # ---------------------------------------
@@ -59,13 +79,12 @@ clean:
 # ---------------------------------------
 # SQUEAKY_CLEAN: remove all of the final targets to make sure we don't leave old artifacts around
 # ---------------------------------------
-squeaky-clean: clean $(patsubst %,squeaky-clean-%,$(PKG_TGTS))
+squeaky-clean: uninstall clean $(patsubst %,squeaky-clean-%,$(PKG_TGTS))
 	find docs/*  ! -name 'README.*' -exec rm -rf {} +
 	find $(PKG_DIR) -name "*.py" ! -name "__init__.py" ! -name "linkml_files.py" -exec rm -f {} +
-	rm -rf .venv
 
 squeaky-clean-%: clean
-	find $(PKG_DIR)/$* ! -name 'README.*' ! -name $*  -type f -exec rm -f {} +
+	find $(PKG_DIR)/$* ! -name model ! -name 'README.*' ! -name $*  -type f -exec rm -f {} +
 
 # ---------------------------------------
 # T: List files to generate
@@ -79,142 +98,156 @@ t:
 echo:
 	echo $(patsubst %,gen-%,$(TGTS))
 
-
 tdir-%:
 	rm -rf target/$*
 	mkdir -p target/$*
 
-docs:
-	mkdir -p $@
-
-
 # ---------------------------------------
 # MARKDOWN DOCS
 #      Generate documentation ready for mkdocs
 # ---------------------------------------
-gen-docs: docs/index.md
+gen-docs: docs/index.html
 .PHONY: gen-docs
 
-docs/index.md: target/docs/index.md
+docs/index.html: target/docs/index.md install
+	echo "HERE!" $(PKG_T_DOCS)
+	mkdir -p $(PKG_T_DOCS)
 	cp -R $(MODEL_DOCS_DIR)/*.md target/docs
-	$(RUN)mkdocs build
+	# mkdocs.yml moves from the target/docs to the docs directory
+	$(RUN) mkdocs build
+
 target/docs/index.md: $(SCHEMA_DIR)/$(SCHEMA_NAME).yaml tdir-docs install
-	$(RUN)gen-markdown $(GEN_OPTS) --mergeimports --notypesdir --warnonexist --dir target/docs $<
+	$(RUN) gen-markdown $(GEN_OPTS) --mergeimports --notypesdir --warnonexist --dir target/docs $<
 
-# ---------------------------------------
-# YAML source
-# ---------------------------------------
-# YAML source is no longewr copied to the output directory.  Instead, it is placed there via setup.cfg
-gen-model:
-.PHONY: gen-model
 
 # ---------------------------------------
-# python source
+# PYTHON Source
 # ---------------------------------------
-gen-python: $(patsubst %, $(PKG_DIR)/%.py, $(SCHEMA_NAMES))
-$(PKG_DIR)/%.py: target/python/%.py
+gen-python: $(patsubst %, $(PKG_T_PYTHON)/%.py, $(SCHEMA_NAMES))
+$(PKG_T_PYTHON)/%.py: target/python/%.py
+	mkdir -p $(PKG_T_PYTHON)
 	cp $< $@
 target/python/%.py: $(SCHEMA_DIR)/%.yaml  tdir-python install
-	$(RUN)gen-python $(GEN_OPTS) --genmeta --no-slots --no-mergeimports $< > $@
+	$(RUN) gen-python $(GEN_OPTS)  --no-slots --no-mergeimports $< > $@
 
 # ---------------------------------------
 # GRAPHQL Source
 # ---------------------------------------
-# TODO: modularize imports. For now imports are merged.
-gen-graphql: $(PKG_DIR)/graphql/$(SCHEMA_NAME).graphql
+gen-graphql: $(PKG_T_GRAPHQL)/$(SCHEMA_NAME).graphql
 .PHONY: gen-graphql
 
-$(PKG_DIR)/graphql/%.graphql: target/graphql/%.graphql
+$(PKG_T_GRAPHQL)/%.graphql: target/graphql/%.graphql
+	mkdir -p $(PKG_T_GRAPHQL)
 	cp $< $@
+
 target/graphql/%.graphql: $(SCHEMA_DIR)/%.yaml tdir-graphql install
-	$(RUN)gen-graphql $(GEN_OPTS) $< > $@
+	$(RUN) gen-graphql $(GEN_OPTS) $< > $@
 
 # ---------------------------------------
 # JSON Schema
 # ---------------------------------------
-gen-jsonschema: $(patsubst %, $(PKG_DIR)/jsonschema/%.schema.json, $(SCHEMA_NAMES))
+gen-jsonschema: $(patsubst %, $(PKG_T_JSON_SCHEMA)/%.schema.json, $(SCHEMA_NAMES))
 .PHONY: gen-jsonschema
-$(PKG_DIR)/jsonschema/%.schema.json: target/jsonschema/%.schema.json
+
+$(PKG_T_JSON_SCHEMA)/%.schema.json: target/json_schema/%.schema.json
+	mkdir -p $(PKG_T_JSON_SCHEMA)
 	cp $< $@
-target/jsonschema/%.schema.json: $(SCHEMA_DIR)/%.yaml tdir-jsonschema install
-	$(RUN)gen-json-schema $(GEN_OPTS) -t transaction $< > $@
+
+target/json_schema/%.schema.json: $(SCHEMA_DIR)/%.yaml tdir-json_schema install
+	$(RUN) gen-json-schema $(GEN_OPTS) -t transaction $< > $@
 
 # ---------------------------------------
 # ShEx
 # ---------------------------------------
-gen-shex: $(patsubst %, $(PKG_DIR)/shex/%.shex, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/shex/%.shexj, $(SCHEMA_NAMES))
+gen-shex: $(patsubst %, $(PKG_T_SHEX)/%.shex, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_SHEX)/%.shexj, $(SCHEMA_NAMES))
 .PHONY: gen-shex
 
-$(PKG_DIR)/shex/%.shex: target/shex/%.shex
+$(PKG_T_SHEX)/%.shex: target/shex/%.shex
+	mkdir -p $(PKG_T_SHEX)
 	cp $< $@
-$(PKG_DIR)/shex/%.shexj: target/shex/%.shexj
+$(PKG_T_SHEX)/%.shexj: target/shex/%.shexj
+	mkdir -p $(PKG_T_SHEX)
 	cp $< $@
 
 target/shex/%.shex: $(SCHEMA_DIR)/%.yaml tdir-shex install
-	$(RUN)gen-shex --no-mergeimports $(GEN_OPTS) $< > $@
+	$(RUN) gen-shex --no-mergeimports $(GEN_OPTS) $< > $@
 target/shex/%.shexj: $(SCHEMA_DIR)/%.yaml tdir-shex install
-	$(RUN)gen-shex --no-mergeimports $(GEN_OPTS) -f json $< > $@
+	$(RUN) gen-shex --no-mergeimports $(GEN_OPTS) -f json $< > $@
 
 # ---------------------------------------
 # OWL
 # ---------------------------------------
-# TODO: modularize imports. For now imports are merged.
-gen-owl: $(PKG_DIR)/owl/$(SCHEMA_NAME).owl.ttl
+gen-owl: $(PKG_T_OWL)/$(SCHEMA_NAME).owl.ttl
 .PHONY: gen-owl
 
-$(PKG_DIR)/owl/%.owl.ttl: target/owl/%.owl.ttl
+$(PKG_T_OWL)/%.owl.ttl: target/owl/%.owl.ttl
+	mkdir -p $(PKG_T_OWL)
 	cp $< $@
 target/owl/%.owl.ttl: $(SCHEMA_DIR)/%.yaml tdir-owl install
-	$(RUN)gen-owl $(GEN_OPTS) $< > $@
+	$(RUN) gen-owl $(GEN_OPTS) $< > $@
 
 # ---------------------------------------
 # JSON-LD Context
 # ---------------------------------------
-gen-jsonld: $(patsubst %, $(PKG_DIR)/jsonld/%.context.jsonld, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/jsonld/%.model.context.jsonld, $(SCHEMA_NAMES)) $(PKG_DIR)/jsonld/context.jsonld
+gen-jsonld: $(patsubst %, $(PKG_T_JSONLD_CONTEXT)/%.context.jsonld, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_JSONLD_CONTEXT)/%.model.context.jsonld, $(SCHEMA_NAMES))
 .PHONY: gen-jsonld
 
-$(PKG_DIR)/jsonld/%.context.jsonld: target/jsonld/%.context.jsonld
-	cp $< $@
-
-$(PKG_DIR)/jsonld/%.model.context.jsonld: target/jsonld/%.model.context.jsonld
+$(PKG_T_JSONLD_CONTEXT)/%.context.jsonld: target/jsonld/%.context.jsonld
+	mkdir -p $(PKG_T_JSONLD_CONTEXT)
 	cp $< $@
 
-$(PKG_DIR)/jsonld/context.jsonld: target/jsonld/meta.context.jsonld
+$(PKG_T_JSONLD_CONTEXT)/%.model.context.jsonld: target/jsonld/%.model.context.jsonld
+	mkdir -p $(PKG_T_JSONLD_CONTEXT)
 	cp $< $@
 
 target/jsonld/%.context.jsonld: $(SCHEMA_DIR)/%.yaml tdir-jsonld install
-	$(RUN)gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@
+	$(RUN) gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@
+
 target/jsonld/%.model.context.jsonld: $(SCHEMA_DIR)/%.yaml tdir-jsonld install
-	$(RUN)gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@
+	$(RUN) gen-jsonld-context $(GEN_OPTS) --no-mergeimports $< > $@
 
 # ---------------------------------------
 # Plain Old (PO) JSON
 # ---------------------------------------
-gen-json: $(patsubst %, $(PKG_DIR)/json/%.json, $(SCHEMA_NAMES))
+gen-json: $(patsubst %, $(PKG_T_JSON)/%.json, $(SCHEMA_NAMES))
 .PHONY: gen-json
 
-$(PKG_DIR)/json/%.json: target/json/%.json
+$(PKG_T_JSON)/%.json: target/json/%.json
+	mkdir -p $(PKG_T_JSON)
 	cp $< $@
 target/json/%.json: $(SCHEMA_DIR)/%.yaml tdir-json install
-	$(RUN)gen-jsonld $(GEN_OPTS) --no-mergeimports $< > $@
+	$(RUN) gen-jsonld $(GEN_OPTS) --no-mergeimports $< > $@
 
 # ---------------------------------------
 # RDF
 # ---------------------------------------
-gen-rdf: gen-jsonld $(patsubst %, $(PKG_DIR)/rdf/%.ttl, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_DIR)/rdf/%.model.ttl, $(SCHEMA_NAMES))
+gen-rdf: gen-jsonld $(patsubst %, $(PKG_T_RDF)/%.ttl, $(SCHEMA_NAMES)) $(patsubst %, $(PKG_T_RDF)/%.model.ttl, $(SCHEMA_NAMES))
 .PHONY: gen-rdf
 
-$(PKG_DIR)/rdf/%.ttl: target/rdf/%.ttl
+$(PKG_T_RDF)/%.ttl: target/rdf/%.ttl
+	mkdir -p $(PKG_T_RDF)
 	cp $< $@
-$(PKG_DIR)/rdf/%.model.ttl: target/rdf/%.model.ttl
+$(PKG_T_RDF)/%.model.ttl: target/rdf/%.model.ttl
+	mkdir -p $(PKG_T_RDF)
 	cp $< $@
 
 target/rdf/%.ttl: $(SCHEMA_DIR)/%.yaml $(PKG_DIR)/jsonld/%.context.jsonld tdir-rdf install
-	$(RUN)gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@
+	$(RUN) gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@
 target/rdf/%.model.ttl: $(SCHEMA_DIR)/%.yaml $(PKG_DIR)/jsonld/%.model.context.jsonld tdir-rdf install
-	$(RUN)gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@
+	$(RUN) gen-rdf $(GEN_OPTS) --context $(realpath $(word 2,$^)) $< > $@
 
+# ---------------------------------------
+# SQLDDL
+# ---------------------------------------
+gen-sqlddl: $(PKG_T_SQLDDL)/$(SCHEMA_NAME).sql
+.PHONY: gen-sqlddl
+
+$(PKG_T_SQLDDL)/%.sql: target/sqlddl/%.sql
+	mkdir -p $(PKG_T_SQLDDL)
+	cp $< $@
+target/sqlddl/%.sql: $(SCHEMA_DIR)/%.yaml tdir-sqlddl install
+	$(RUN) gen-sqlddl $(GEN_OPTS) $< > $@
 
 # test docs locally.
-docserve: gen-docs install
-	$(RUN)mkdocs serve
+docserve: gen-docs
+	$(RUN) mkdocs serve

Pipfile

@@ -5,6 +5,8 @@ name = "pypi"
 
 [packages]
 linkml-runtime = "~=1.0, >=1.0.10"
+linkml = "~=1.0"
+mkdocs = "*"
 
 [dev-packages]
 linkml = "~=1.0, >=1.0.1"

hide_test_changes.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 # Make all of the test output files invisible to git
-echo git update-index --assume-unchanged `git status -s docs | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//'`
-echo git update-index --assume-unchanged `git status -s linkml_model | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//' | grep -v \/model\/`
+git update-index --assume-unchanged `git status -s docs | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//'`
+git update-index --assume-unchanged `git status -s linkml_model | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*//' | grep -v \/model\/`