From 019f211cf53bf34c0fa513b09afbcab4edfe146c Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 22 Nov 2022 17:05:14 -0800 Subject: [PATCH 1/3] First pass at PyOBO ingest. See #45 --- Makefile | 3 +- ontologies.Makefile | 63 +++++++++++++++++-- src/semsql/builder/builder.py | 7 ++- src/semsql/builder/prefixes/prefixes.csv | 6 ++ .../builder/prefixes/prefixes_local.csv | 6 ++ src/semsql/builder/registry/ontologies.yaml | 31 ++++++++- 6 files changed, 108 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index d5aefcb..c65d2af 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ BUILDER_DIR = src/semsql/builder DDL_DIR = $(BUILDER_DIR)/sql_schema YAML_DIR = src/semsql/linkml SQLA_DIR = src/semsql/sqla +ONT_REGISTRY = src/semsql/builder/registry/ontologies.yaml PREFIX_DIR = $(BUILDER_DIR)/prefixes @@ -141,7 +142,7 @@ download/reactome-biopax.zip: src/semsql/builder/registry/registry_schema.py: src/semsql/builder/registry/registry_schema.yaml $(RUN) gen-python $< > $@ -ontologies.Makefile: src/semsql/builder/registry/ontologies.yaml +ontologies.Makefile: $(ONT_REGISTRY) $(RUN) semsql generate-makefile -P src/semsql/builder/prefixes/prefixes_local.csv $< > $@.tmp && mv $@.tmp $@ include ontologies.Makefile diff --git a/ontologies.Makefile b/ontologies.Makefile index ce1f9cd..40395d9 100644 --- a/ontologies.Makefile +++ b/ontologies.Makefile @@ -208,7 +208,7 @@ db/mlo.owl: download/mlo.owl download/ito.owl: STAMP - curl -L -s https://github.com/OpenBioLink/ITO/raw/master/ITO.owl.zip > $@.zip.tmp && unzip -p $@.zip.tmp ITO.owl > $@.tmp && rm $@.zip.tmp + curl -L -s https://github.com/OpenBioLink/ITO/raw/master/ITO.owl.zip > $@.zip.tmp && unzip -p $@.zip.tmp {ont.zip_extract_file} > $@.tmp && rm $@.zip.tmp sha256sum -b $@.tmp > $@.sha256 mv $@.tmp $@ @@ -219,7 +219,7 @@ db/ito.owl: download/ito.owl download/reactome-Homo-sapiens.owl: STAMP - curl -L -s https://reactome.org/download/current/biopax.zip > $@.zip.tmp && unzip -p $@.zip.tmp Homo_sapiens.owl > $@.tmp && rm $@.zip.tmp + curl -L -s https://reactome.org/download/current/biopax.zip > $@.zip.tmp && unzip -p $@.zip.tmp {ont.zip_extract_file} > $@.tmp && rm $@.zip.tmp sha256sum -b $@.tmp > $@.sha256 mv $@.tmp $@ @@ -263,7 +263,7 @@ db/sweetAll.owl: download/sweetAll.owl download/lov.owl: STAMP - curl -L -s https://lov.linkeddata.es/lov.n3.gz > $@.tmp + curl -L -s https://lov.linkeddata.es/lov.n3.gz | gzip -dc > $@.tmp sha256sum -b $@.tmp > $@.sha256 mv $@.tmp $@ @@ -306,6 +306,61 @@ db/co_324.owl: download/co_324.owl cp $< $@ +download/hgnc.genegroup.owl: STAMP + curl -L -s https://github.com/biopragmatics/obo-db-ingest/raw/main/export/hgnc.genegroup/hgnc.genegroup.owl.gz | gzip -dc > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/hgnc.genegroup.owl + +db/hgnc.genegroup.owl: download/hgnc.genegroup.owl + cp $< $@ + + +download/hgnc.owl: STAMP + curl -L -s https://github.com/biopragmatics/obo-db-ingest/raw/main/export/hgnc/2022-06-01/hgnc.owl.gz | gzip -dc > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/hgnc.owl + +db/hgnc.owl: download/hgnc.owl + cp $< $@ + + +download/dictybase.owl: STAMP + curl -L -s https://github.com/biopragmatics/obo-db-ingest/raw/main/export/dictybase/dictybase.owl.gz | gzip -dc > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/dictybase.owl + +db/dictybase.owl: download/dictybase.owl + cp $< $@ + + +download/eccode.owl: STAMP + curl -L -s https://github.com/biopragmatics/obo-db-ingest/raw/main/export/eccode/25-May-2022/eccode.owl.gz | gzip -dc > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/eccode.owl + +db/eccode.owl: download/eccode.owl + cp $< $@ + + +download/uniprot.owl: STAMP + curl -L -s https://github.com/biopragmatics/obo-db-ingest/raw/main/export/uniprot/2022_02/uniprot.owl.gz | gzip -dc > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/uniprot.owl + +db/uniprot.owl: download/uniprot.owl + cp $< $@ + + download/%.owl: STAMP curl -L -s http://purl.obolibrary.org/obo/$*.owl > $@.tmp sha256sum -b $@.tmp > $@.sha256 @@ -316,4 +371,4 @@ download/%.owl: STAMP db/%.owl: download/%.owl robot merge -i $< -o $@ -EXTRA_ONTOLOGIES = chiro ncit foodon chebiplus msio phenio comploinc bero aio reacto go go-lego bao orcid cpont biolink biopax enanomapper mlo ito reactome-Homo-sapiens efo edam sweetAll lov schema-dot-org cosmo co_324 +EXTRA_ONTOLOGIES = chiro ncit foodon chebiplus msio phenio comploinc bero aio reacto go go-lego bao orcid cpont biolink biopax enanomapper mlo ito reactome-Homo-sapiens efo edam sweetAll lov schema-dot-org cosmo co_324 hgnc.genegroup hgnc dictybase eccode uniprot diff --git a/src/semsql/builder/builder.py b/src/semsql/builder/builder.py index 02d5ca8..3daaebb 100644 --- a/src/semsql/builder/builder.py +++ b/src/semsql/builder/builder.py @@ -13,7 +13,7 @@ from semsql.builder.registry import registry_schema from semsql.builder.registry.registry_schema import (Makefile, MakefileRule, - Ontology) + Ontology, CompressionEnum) from semsql.utils.makefile_utils import makefile_to_string this_path = Path(__file__).parent @@ -137,6 +137,11 @@ def compile_registry(registry_path: str, local_prefix_file: TextIO = None) -> st "unzip -p $@.zip.tmp {ont.zip_extract_file} " "> $@.tmp && rm $@.zip.tmp" ) + elif ont.compression: + if str(ont.compression) == str(CompressionEnum.gzip.text): + command = f"curl -L -s {ont.url} | gzip -dc > $@.tmp" + else: + raise ValueError(f"Unknown compression: '{ont.compression}'") else: command = f"curl -L -s {ont.url} > $@.tmp" download_rule = MakefileRule( diff --git a/src/semsql/builder/prefixes/prefixes.csv b/src/semsql/builder/prefixes/prefixes.csv index 5a58706..3c8ff58 100644 --- a/src/semsql/builder/prefixes/prefixes.csv +++ b/src/semsql/builder/prefixes/prefixes.csv @@ -59,6 +59,12 @@ evs.ncit,http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl# old.fix,http://purl.org/obo/owl/FIX# mlo,http://www.a2rd.net.br/mlo# co_324,https://cropontology.org/rdf/CO_324: +hgnc.genegroup,http://purl.obolibrary.org/obo/hgnc.genegroup_ +hgnc,http://purl.obolibrary.org/obo/hgnc_ +hgnc.genegroup,http://purl.obolibrary.org/obo/dictybase_ +EC,http://purl.obolibrary.org/obo/eccode_ +uniprot.obo,http://purl.obolibrary.org/obo/uniprot_ +uniprot.obo,http://purl.obolibrary.org/obo/uniprot_ RBO,http://purl.obolibrary.org/obo/RBO_ CLYH,http://purl.obolibrary.org/obo/CLYH_ RO,http://purl.obolibrary.org/obo/RO_ diff --git a/src/semsql/builder/prefixes/prefixes_local.csv b/src/semsql/builder/prefixes/prefixes_local.csv index c506c6a..df21910 100644 --- a/src/semsql/builder/prefixes/prefixes_local.csv +++ b/src/semsql/builder/prefixes/prefixes_local.csv @@ -7,3 +7,9 @@ evs.ncit,http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl# old.fix,http://purl.org/obo/owl/FIX# mlo,http://www.a2rd.net.br/mlo# co_324,https://cropontology.org/rdf/CO_324: +hgnc.genegroup,http://purl.obolibrary.org/obo/hgnc.genegroup_ +hgnc,http://purl.obolibrary.org/obo/hgnc_ +hgnc.genegroup,http://purl.obolibrary.org/obo/dictybase_ +EC,http://purl.obolibrary.org/obo/eccode_ +uniprot.obo,http://purl.obolibrary.org/obo/uniprot_ +uniprot.obo,http://purl.obolibrary.org/obo/uniprot_ diff --git a/src/semsql/builder/registry/ontologies.yaml b/src/semsql/builder/registry/ontologies.yaml index c7224a3..2f9053b 100644 --- a/src/semsql/builder/registry/ontologies.yaml +++ b/src/semsql/builder/registry/ontologies.yaml @@ -110,5 +110,32 @@ ontologies: url: https://cropontology.org/ontology/CO_324/rdf prefixmap: co_324: "https://cropontology.org/rdf/CO_324:" - - + +## PyOBO products +## See https://github.com/INCATools/semantic-sql/issues/45 + hgnc.genegroup: + url: https://github.com/biopragmatics/obo-db-ingest/raw/main/export/hgnc.genegroup/hgnc.genegroup.owl.gz + compression: gzip + prefixmap: + hgnc.genegroup: http://purl.obolibrary.org/obo/hgnc.genegroup_ + hgnc: + url: https://github.com/biopragmatics/obo-db-ingest/raw/main/export/hgnc/2022-06-01/hgnc.owl.gz + compression: gzip + prefixmap: + hgnc: http://purl.obolibrary.org/obo/hgnc_ + dictybase: + url: https://github.com/biopragmatics/obo-db-ingest/raw/main/export/dictybase/dictybase.owl.gz + compression: gzip + prefixmap: + hgnc.genegroup: http://purl.obolibrary.org/obo/dictybase_ + eccode: + url: https://github.com/biopragmatics/obo-db-ingest/raw/main/export/eccode/25-May-2022/eccode.owl.gz + compression: gzip + prefixmap: + EC: http://purl.obolibrary.org/obo/eccode_ + uniprot.obo: http://purl.obolibrary.org/obo/uniprot_ + uniprot: + url: https://github.com/biopragmatics/obo-db-ingest/raw/main/export/uniprot/2022_02/uniprot.owl.gz + compression: gzip + prefixmap: + uniprot.obo: http://purl.obolibrary.org/obo/uniprot_ From 6e85c0abdd427ee0ffa11da03634a0d0e1a899aa Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 22 Nov 2022 17:14:25 -0800 Subject: [PATCH 2/3] lint --- src/semsql/builder/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/semsql/builder/builder.py b/src/semsql/builder/builder.py index 3daaebb..f4e10c3 100644 --- a/src/semsql/builder/builder.py +++ b/src/semsql/builder/builder.py @@ -12,8 +12,8 @@ from sqlalchemy.orm import sessionmaker from semsql.builder.registry import registry_schema -from semsql.builder.registry.registry_schema import (Makefile, MakefileRule, - Ontology, CompressionEnum) +from semsql.builder.registry.registry_schema import (CompressionEnum, Makefile, + MakefileRule, Ontology) from semsql.utils.makefile_utils import makefile_to_string this_path = Path(__file__).parent From 4fb7bac23d13eaa0c3791799011a573817371083 Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 6 Dec 2022 14:32:11 -0800 Subject: [PATCH 3/3] Adding reason step for maxo --- Makefile | 9 ++++++--- ontologies.Makefile | 13 ++++++++++++- src/semsql/builder/registry/ontologies.yaml | 4 ++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c65d2af..b47c99d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ SELECTED_ONTS = obi mondo go envo ro hp mp zfa wbphenotype ecto upheno uberon_cm # EXTRA_ONTOLOGIES is defined in ontologies.Makefile ALL_ONTS = $(ALL_OBO_ONTS) $(EXTRA_ONTOLOGIES) -STAGED_ONTOLOGIES = $(patsubst %,stage/%.db.gz,$(ALL_ONTS)) +STAGED_ONTOLOGIES = $(patsubst %, stage/%.db.gz, $(ALL_ONTS)) TEST_ONTOLOGIES = go-nucleus robot-example @@ -25,13 +25,16 @@ TEST_ONTOLOGIES = go-nucleus robot-example all: build_all stage_all build_all: $(patsubst %,all-%,$(ALL_ONTS)) stage_all: $(STAGED_ONTOLOGIES) + echo done $(STAGED_ONTOLOGIES) selected: $(patsubst %,all-%,$(SELECTED_ONTS)) all-%: db/%.db sqlite3 $< "SELECT COUNT(*) FROM statements" -stage/%.db.gz: db/%.db - gzip -c $< > $@.tmp && mv $@.tmp $@ +#stage/%.db.gz: db/%.db +# gzip -c $< > $@.tmp && mv $@.tmp $@ +stage/%.db.gz: + gzip -c db/$*.db > $@.tmp && mv $@.tmp $@ .PRECIOUS: stage/%.db.gz list-onts: diff --git a/ontologies.Makefile b/ontologies.Makefile index 40395d9..ff655a3 100644 --- a/ontologies.Makefile +++ b/ontologies.Makefile @@ -20,6 +20,17 @@ db/ncit.owl: download/ncit.owl robot relax -i $< merge -o $@ +download/maxo.owl: STAMP + curl -L -s http://purl.obolibrary.org/obo/maxo.owl > $@.tmp + sha256sum -b $@.tmp > $@.sha256 + mv $@.tmp $@ + +.PRECIOUS: download/maxo.owl + +db/maxo.owl: download/maxo.owl + robot relax -i $< merge -o $@ + + download/foodon.owl: STAMP curl -L -s http://purl.obolibrary.org/obo/foodon.owl > $@.tmp sha256sum -b $@.tmp > $@.sha256 @@ -371,4 +382,4 @@ download/%.owl: STAMP db/%.owl: download/%.owl robot merge -i $< -o $@ -EXTRA_ONTOLOGIES = chiro ncit foodon chebiplus msio phenio comploinc bero aio reacto go go-lego bao orcid cpont biolink biopax enanomapper mlo ito reactome-Homo-sapiens efo edam sweetAll lov schema-dot-org cosmo co_324 hgnc.genegroup hgnc dictybase eccode uniprot +EXTRA_ONTOLOGIES = chiro ncit maxo foodon chebiplus msio phenio comploinc bero aio reacto go go-lego bao orcid cpont biolink biopax enanomapper mlo ito reactome-Homo-sapiens efo edam sweetAll lov schema-dot-org cosmo co_324 hgnc.genegroup hgnc dictybase eccode uniprot diff --git a/src/semsql/builder/registry/ontologies.yaml b/src/semsql/builder/registry/ontologies.yaml index 2f9053b..5f98321 100644 --- a/src/semsql/builder/registry/ontologies.yaml +++ b/src/semsql/builder/registry/ontologies.yaml @@ -8,6 +8,10 @@ ontologies: ncit: url: http://purl.obolibrary.org/obo/ncit.owl build_command: "robot relax -i $< merge -o $@" + maxo: + url: http://purl.obolibrary.org/obo/maxo.owl + # https://github.com/monarch-initiative/MAxO/issues/367 + build_command: "robot relax -i $< merge reason -r structural -o $@" foodon: url: http://purl.obolibrary.org/obo/foodon.owl build_command: "robot merge -i $< relax reduce -c true -o $@"